Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: update crawlers to use the new request method (#1281)
NTFSvolume8172948921e4
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pyindex d06cfd3a6..5f059ab76 100644--- a/cyberdrop_dl/clients/flaresolverr.py+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -4,7 +4,6 @@ import asyncioimport dataclassesimport itertoolsimport time-from functools import partialfrom http.cookies import SimpleCookiefrom typing import TYPE_CHECKING, Any@@ -76,7 +75,7 @@ class FlareSolverr:self.manager = managerself._session_id: str = ""self._session_lock, self._request_lock = asyncio.Lock(), asyncio.Lock()- self._next_request_id: Callable[[], int] = partial(next, itertools.count(1))+ self._next_request_id: Callable[[], int] = itertools.count(1).__next__if manager.global_config.general.flaresolverr:self.url = manager.global_config.general.flaresolverr / "v1"else:@@ -89,8 +88,6 @@ class FlareSolverr:await self._destroy_session()async def request(self, url: AbsoluteHttpURL, data: Any = None) -> FlareSolverrSolution:- # TODO: make this method return an abstract response-invalid_response_error = DDOSGuardError("Invalid response from flaresolverr")try:if not self._session_id:diff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.pyindex d4e0a97f5..fc4b27d00 100644--- a/cyberdrop_dl/clients/response.py+++ b/cyberdrop_dl/clients/response.py@@ -3,12 +3,16 @@ from __future__ import annotationsimport asyncioimport dataclassesfrom json import loads as json_loads+from types import MappingProxyTypefrom typing import TYPE_CHECKING, Any, Self+import aiohttp.multipartfrom aiohttp import ClientResponse+from aiohttp.client_reqrep import ContentDispositionfrom aiohttp_client_cache.response import CachedResponsefrom bs4 import BeautifulSoupfrom multidict import CIMultiDict, CIMultiDictProxy+from propcache import under_cached_propertyfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import InvalidContentTypeError, ScrapeError@@ -22,10 +26,12 @@ if TYPE_CHECKING:@dataclasses.dataclass(slots=True, weakref_slot=True)class AbstractResponse:- """Class to represent common methods and attributes between:- - `aiohttp.ClientResponse`- - `curl_cffi.Response`- - `FlareSolverrSolution`"""+ """+ Class to represent common methods and attributes between:+ - `aiohttp.ClientResponse`+ - `curl_cffi.Response`+ - `FlareSolverrSolution`+ """content_type: strstatus: int@@ -35,6 +41,7 @@ class AbstractResponse:_resp: ClientResponse | CachedResponse | CurlResponse | None = None_text: str = ""+ _cache: dict[str, Any] = dataclasses.field(init=False, default_factory=dict)_read_lock: asyncio.Lock = dataclasses.field(init=False, default_factory=asyncio.Lock)@classmethod@@ -44,7 +51,8 @@ class AbstractResponse:headers = response.headerselse:status = response.status_code- headers = CIMultiDictProxy(CIMultiDict({k: v or "" for k, v in response.headers}))+ multi_items = ((k, v) for k, v in response.headers.multi_items() if v is not None)+ headers = CIMultiDictProxy(CIMultiDict(multi_items))url = AbsoluteHttpURL(response.url)content_type, location = cls.parse_headers(url, headers)@@ -70,7 +78,7 @@ class AbstractResponse:)@staticmethod- def parse_headers(url: AbsoluteHttpURL, headers: CIMultiDictProxy[str]):+ def parse_headers(url: AbsoluteHttpURL, headers: CIMultiDictProxy[str]) -> tuple[str, AbsoluteHttpURL | None]:if location := headers.get("location"):location = parse_url(location, url.origin(), trim=False)else:@@ -79,6 +87,38 @@ class AbstractResponse:content_type = (headers.get("Content-Type") or "").lower()return content_type, location+ @under_cached_property+ def content_disposition(self) -> ContentDisposition:+ header = self.headers["Content-Disposition"]+ disposition_type, params = aiohttp.multipart.parse_content_disposition(header)+ params = MappingProxyType(params)+ filename = aiohttp.multipart.content_disposition_filename(params)+ return ContentDisposition(disposition_type, params, filename)++ @property+ def filename(self) -> str:+ assert self.content_disposition.filename+ return self.content_disposition.filename++ @property+ def consumed(self) -> bool:+ return bool(self._text)++ @property+ def ok(self) -> bool:+ """Returns `True` if `status` is less than `400`, `False` if not.++ This is **not** a check for ``200 OK``+ """+ return self.status < 400++ async def read(self) -> bytes:+ assert self._resp is not None+ async with self._read_lock:+ if isinstance(self._resp, ClientResponse | CachedResponse):+ return await self._resp.read()+ return await self._resp.acontent()+async def text(self, encoding: str | None = None) -> str:if self._text:return self._text@@ -119,4 +159,4 @@ class AbstractResponse:raise InvalidContentTypeError(message=msg)def __repr__(self) -> str:- return f"<{self.__class__.__name__} [{self.status}] ({self.url!r})>"+ return f"<{self.__class__.__name__} [{self.status}] ({self.url})>"diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pyindex e00721edf..0c49e1c94 100644--- a/cyberdrop_dl/clients/scraper_client.py+++ b/cyberdrop_dl/clients/scraper_client.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport contextlib+import timefrom datetime import datetimefrom json import dumps as json_dumpsfrom pathlib import Path@@ -10,6 +11,7 @@ from typing import TYPE_CHECKING, Any, castimport cyberdrop_dl.constants as constantsfrom cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.exceptions import DDOSGuardError+from cyberdrop_dl.utils.cookie_management import make_simple_cookiefrom cyberdrop_dl.utils.utilities import sanitize_filenameif TYPE_CHECKING:@@ -78,6 +80,19 @@ class ScraperClient:finally:await self.write_soup_to_disk(url, resp, exc)+ def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:+ """+ Apply to the cookies from the `curl` session into the `aiohttp` session, filtering them by the URL++ This is mostly just to get the `cf_cleareance` cookie value into the `aiohttp` session++ The reverse (sync `aiohttp` -> `curl`) is not needed at the moment, so it is skipped+ """+ now = time.time()+ for cookie in self.client_manager._curl_session.cookies.jar:+ simple_cookie = make_simple_cookie(cookie, now)+ self.client_manager.cookies.update_cookies(simple_cookie, url)+@contextlib.asynccontextmanagerasync def __request_context(self,@@ -95,8 +110,7 @@ class ScraperClient:curl_resp = await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)try:yield AbstractResponse.from_resp(curl_resp)- curl_cookies = self.client_manager._curl_session.cookies.get_dict(url.host)- self.client_manager.cookies.update_cookies(curl_cookies, url)+ self.__sync_session_cookies(url)finally:await curl_resp.aclose()return@@ -121,15 +135,14 @@ class ScraperClient:async def write_soup_to_disk(self, url: AbsoluteHttpURL, response: AbstractResponse, exc: Exception | None = None):"""Writes html to a file."""- if not self._save_pages_html:+ if not (+ self._save_pages_html+ and "html" in response.content_type+ and response.consumed # Do not consume the response if the crawler didn't+ ):return- content: str = await response.text()- try:- content = cast("str", (await response.soup()).prettify(formatter="html"))- except Exception:- pass-+ content = cast("str", (await response.soup()).prettify(formatter="html"))now = datetime.now()log_date = now.strftime(constants.LOGS_DATETIME_FORMAT)url_str = str(url)diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.pyindex a062d8c80..d8a98d283 100755--- a/cyberdrop_dl/config/config_model.py+++ b/cyberdrop_dl/config/config_model.py@@ -9,7 +9,6 @@ from pydantic import BaseModel, ByteSize, NonNegativeInt, PositiveInt, field_serfrom cyberdrop_dl import constantsfrom cyberdrop_dl.constants import BROWSERS, DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGEfrom cyberdrop_dl.data_structures.hash import Hashing-from cyberdrop_dl.data_structures.supported_domains import SUPPORTED_SITES_DOMAINSfrom cyberdrop_dl.models import HttpAppriseURLfrom cyberdrop_dl.models.types import (ByteSizeSerilized,@@ -22,6 +21,7 @@ from cyberdrop_dl.models.types import (PathOrNone,)from cyberdrop_dl.models.validators import falsy_as, to_timedelta+from cyberdrop_dl.supported_domains import SUPPORTED_SITES_DOMAINSfrom cyberdrop_dl.utils.strings import validate_format_stringfrom cyberdrop_dl.utils.utilities import purge_dir_treediff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex cebbb1efc..d8c01c476 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -149,7 +149,9 @@ class CheveretoCrawler(Crawler, is_generic=True):raise PasswordProtectedErrorsoup = await self.request_soup(- _sort_by_new(scrape_item.url / ""), method="POST", data={"content-password": password}+ _sort_by_new(scrape_item.url / ""),+ method="POST",+ data={"content-password": password},)if _is_password_protected(soup):@@ -167,9 +169,7 @@ class CheveretoCrawler(Crawler, is_generic=True):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup = await self.request_soup(scrape_item.url)-+ soup = await self.request_soup(scrape_item.url)link_str = open_graph.get("video", soup) or open_graph.get("image", soup)if not link_str or "loading.svg" in link_str:link_str = Selector.MAIN_IMAGE(soup)diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex bd49b4d18..64c80a032 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -25,7 +25,7 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import LoginError, MaxChildrenError, ScrapeErrorfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.dates import TimeStamp, to_timestamp-from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, is_absolute_http_url, is_blob_or_svg+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, is_blob_or_svgif TYPE_CHECKING:from collections.abc import AsyncGenerator, Iterable, Sequence@@ -178,17 +178,17 @@ class MessageBoardCrawler(Crawler, is_abc=True):def parse_thread(cls, url: AbsoluteHttpURL, thread_name_and_id: str) -> ThreadProtocol: ...@abstractmethod- async def post(self, scrape_item: ScrapeItem, post: ForumPostProtocol) -> None: ...+ async def post(self, scrape_item: ScrapeItem, /, post: ForumPostProtocol) -> None: ...@abstractmethod- async def thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None: ...+ async def thread(self, scrape_item: ScrapeItem, /, thread: ThreadProtocol) -> None: ...async def forum(self, scrape_item: ScrapeItem) -> None:# Subclasses can define custom logic for this method.# They would need to also override `fetch` since the default fetch does not take this method into accountraise NotImplementedError- async def resolve_confirmation_link(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:+ async def resolve_confirmation_link(self, url: AbsoluteHttpURL, /) -> AbsoluteHttpURL | None:# Not every forum has confirmation link so overriding this method is optional# Implementation of this method MUST return `None` instead of raising an errorraise NotImplementedError@@ -251,27 +251,6 @@ class MessageBoardCrawler(Crawler, is_abc=True):by_host = any(host in link.host for host in self.ATTACHMENT_HOSTS)return by_parts or by_host- # TODO: Move this to base crawler- @final- @error_handling_wrapper- async def follow_redirect_w_get(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- response, _ = await self.client._get_response_and_soup(self.DOMAIN, scrape_item.url)- assert is_absolute_http_url(response.url)- scrape_item.url = response.url- self.manager.task_group.create_task(self.run(scrape_item))-- # TODO: Move this to base crawler- @final- @error_handling_wrapper- async def follow_redirect_w_head(self, scrape_item: ScrapeItem) -> None:- head = await self.client.get_head(self.DOMAIN, scrape_item.url)- redirect = head.get("location")- if not redirect:- raise ScrapeError(422)- scrape_item.url = self.parse_url(redirect)- self.manager.task_group.create_task(self.run(scrape_item))-@finalasync def follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:url = await self.resolve_confirmation_link(scrape_item.url)@@ -310,7 +289,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):if self.is_attachment(link):return await self.handle_internal_link(scrape_item, link)if self.PRIMARY_URL.host == link.host:- self.manager.task_group.create_task(self.run(scrape_item.create_child(link)))+ self.create_task(self.run(scrape_item.create_child(link)))returnnew_scrape_item = scrape_item.create_child(link)self.handle_external_links(new_scrape_item)@@ -353,8 +332,9 @@ class MessageBoardCrawler(Crawler, is_abc=True):self.log(msg, 30)async def check_login_with_request(self, login_url: AbsoluteHttpURL) -> tuple[str, bool]:- text = await self.client.get_text(self.DOMAIN, login_url, cache_disabled=True)- return text, any(p in text for p in ('<span class="p-navgroup-user-linkText">', "You are already logged in."))+ text = await self.request_text(login_url, cache_disabled=True)+ logged_in = '<span class="p-navgroup-user-linkText">' in text or "You are already logged in." in text+ return text, logged_inclass HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):@@ -417,7 +397,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):return thread.url / f"{cls.POST_URL_PART_NAME}-{post_id}"@error_handling_wrapper- async def thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None:+ async def thread(self, scrape_item: ScrapeItem, /, thread: ThreadProtocol) -> None:scrape_item.setup_as_forum("")if thread.url in self.scraped_threads:return@@ -465,7 +445,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):possible_datetime=current_post.timestamp,add_parent=post_url,)- self.manager.task_group.create_task(self.post(new_scrape_item, current_post))+ self.create_task(self.post(new_scrape_item, current_post))try:scrape_item.add_children()except MaxChildrenError:@@ -582,12 +562,12 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):if url.startswith("https://"):return self.parse_url(url)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, link)+ soup = await self.request_soup(link)selector = self.SELECTORS.confirmation_buttonconfirm_button = soup.select_one(selector.element)if not confirm_button:return+link_str: str = css.get_attr(confirm_button, selector.attribute)link_str = link_str.split('" class="link link--internal', 1)[0]new_link = self.parse_url(link_str)diff --git a/cyberdrop_dl/crawlers/_kemono_base.py b/cyberdrop_dl/crawlers/_kemono_base.pyindex 11a4789bb..572c716fa 100644--- a/cyberdrop_dl/crawlers/_kemono_base.py+++ b/cyberdrop_dl/crawlers/_kemono_base.py@@ -283,7 +283,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):for channel in server.channels:url = self.PRIMARY_URL / "discord/server" / server_id / channel.idnew_scrape_item = scrape_item.create_child(url)- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))scrape_item.add_children()async def discord_channel(self, scrape_item: ScrapeItem, channel_id: str) -> None:@@ -383,7 +383,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):url = self.PRIMARY_URL / service / "user" / user_idnew_scrape_item = scrape_item.create_child(url)- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))# ~~~~~~~~ INTERNAL METHODS, not expected to be overriden, but could be ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex 360f945ae..08b0999cd 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -5,8 +5,6 @@ from __future__ import annotationsimport refrom typing import TYPE_CHECKING, ClassVar, NamedTuple-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError@@ -60,9 +58,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):}NEXT_PAGE_SELECTOR: ClassVar[str] = _SELECTORS.NEXT_PAGE-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(3, 10)+ _RATE_LIMIT = 3, 10async def fetch(self, scrape_item: ScrapeItem) -> None:if any(p in scrape_item.url.parts for p in ("categories", "tags")) or scrape_item.url.query.get("q"):@@ -79,8 +75,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def search(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)title = ""if (search_query := scrape_item.url.query.get("q")) or scrape_item.url.parts[1] == "search":search_query = search_query or scrape_item.url.parts[2]@@ -100,8 +95,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def profile(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)user_name: str = css.select_one_get_text(soup, _SELECTORS.USER_NAME).split("'s Profile")[0].strip()title = f"{user_name} [user]"@@ -119,16 +113,14 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):url = scrape_item.url / video_category if video_category else scrape_item.urlasync for soup in self.web_pager(url):for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEOS):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)-+ soup = await self.request_soup(scrape_item.url)video = self.get_video_info(soup)filename, ext = self.get_filename_and_ext(video.url.name)custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=video.res)@@ -140,25 +132,22 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)-- js_text: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_ID)- album_id: str = get_text_between(js_text, "params['album_id'] =", ";").strip()+ soup = await self.request_soup(scrape_item.url)+ js_text = css.select_one_get_text(soup, _SELECTORS.ALBUM_ID)+ album_id = get_text_between(js_text, "params['album_id'] =", ";").strip()results = await self.get_album_results(album_id)- title: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_NAME)+ title = css.select_one_get_text(soup, _SELECTORS.ALBUM_NAME)title = self.create_title(f"{title} [album]", album_id)scrape_item.setup_as_album(title, album_id=album_id)for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUM_PICTURES, results=results):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def picture(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)url = self.parse_url(css.select_one_get_attr(soup, _SELECTORS.PICTURE, "src"))filename, ext = self.get_filename_and_ext(url.name)await self.handle_file(url, scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/_one_manager.py b/cyberdrop_dl/crawlers/_one_manager.pyindex cd8e645da..b8c012143 100644--- a/cyberdrop_dl/crawlers/_one_manager.py+++ b/cyberdrop_dl/crawlers/_one_manager.py@@ -50,8 +50,7 @@ class OneManagerCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def process_path(self, scrape_item: ScrapeItem) -> None:try:- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)except InvalidContentTypeError: # This is a file, not htmlscrape_item.parent_title = scrape_item.parent_title.rsplit("/", 1)[0]link = scrape_item.url@@ -71,7 +70,7 @@ class OneManagerCrawler(Crawler, is_abc=True):for folder in css.iselect(table, _SELECTORS.FOLDER):link = scrape_item.url / css.select_one_get_attr(folder, _SELECTORS.FOLDER_LINK, "href")new_scrape_item = scrape_item.create_child(link, new_title_part=link.name)- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))scrape_item.add_children()@error_handling_wrapperdiff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.pyindex 3b84c59c9..05c64aaa3 100644--- a/cyberdrop_dl/crawlers/_yetishare.py+++ b/cyberdrop_dl/crawlers/_yetishare.py@@ -3,7 +3,6 @@ from __future__ import annotationsimport itertoolsfrom typing import TYPE_CHECKING, Any, ClassVar-from aiolimiter import AsyncLimiterfrom bs4 import BeautifulSoupfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id@@ -46,6 +45,7 @@ class YetiShareCrawler(Crawler, is_abc=True):),"Shared folders": "/shared/<share_key>",}+ _RATE_LIMIT = 5, 1def __init_subclass__(cls, **kwargs) -> None:super().__init_subclass__(**kwargs)@@ -53,9 +53,6 @@ class YetiShareCrawler(Crawler, is_abc=True):cls.FILE_API_URL = cls.PRIMARY_URL / "account/ajax/file_details"cls.FOLDER_PASSWORD_API_URL = cls.PRIMARY_URL / "ajax/folder_password_process"- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(5, 1)-async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case ["folder", folder_id, *_]:@@ -72,8 +69,7 @@ class YetiShareCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def folder(self, scrape_item: ScrapeItem, folder_id: str, is_shared: bool = False) -> None:# Make request to update cookies. Access to folders is saved in cookies- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)if soup.select_one(Selector.LOGIN_FORM):raise ScrapeError(410, "Folder has been deleted")@@ -133,9 +129,7 @@ class YetiShareCrawler(Crawler, is_abc=True):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)-+ soup = await self.request_soup(scrape_item.url)if soup.select_one(Selector.PASSWORD_PROTECTED):soup = await self._unlock_password_protected_file(scrape_item, file_id)@@ -168,7 +162,8 @@ class YetiShareCrawler(Crawler, is_abc=True):link = self.parse_url(raw_link).with_query(download_token=token)scrape_item.possible_datetime = self.parse_date(- css.select_one_get_text(soup, Selector.FILE_UPLOAD_DATE), "%d/%m/%Y %H:%M:%S"+ css.select_one_get_text(soup, Selector.FILE_UPLOAD_DATE),+ "%d/%m/%Y %H:%M:%S",)filename = css.select_one_get_text(soup, Selector.FILE_NAME)@@ -182,13 +177,12 @@ class YetiShareCrawler(Crawler, is_abc=True):) -> BeautifulSoup:async def ajax_api_request() -> BeautifulSoup:ajax_url = self.FILE_API_URL if is_file else self.FOLDERS_API_URL- async with self.request_limiter:- json_resp: dict[str, str] = await self.client.post_data(- self.DOMAIN,- ajax_url,- data=data,- headers={"X-Requested-With": "XMLHttpRequest"},- )+ json_resp: dict[str, str] = await self.request_json(+ ajax_url,+ method="POST",+ data=data,+ headers={"X-Requested-With": "XMLHttpRequest"},+ )return BeautifulSoup(json_resp["html"].replace("\\", ""), "html.parser")@@ -207,13 +201,16 @@ class YetiShareCrawler(Crawler, is_abc=True):raise PasswordProtectedErrorpassword_post_url = (self.PRIMARY_URL / file_id).with_query("pt=")- async with self.request_limiter:- resp_bytes = await self.client.post_data_raw(- self.DOMAIN,- password_post_url,- data={"filePassword": password, "submitme": 1},- )- soup = BeautifulSoup(resp_bytes, "html.parser")++ content = await self.request_text(+ password_post_url,+ method="POST",+ data={+ "filePassword": password,+ "submitme": 1,+ },+ )+ soup = BeautifulSoup(content, "html.parser")if soup.select_one(Selector.PASSWORD_PROTECTED):raise PasswordProtectedError("File password is invalid")@@ -225,16 +222,15 @@ class YetiShareCrawler(Crawler, is_abc=True):raise PasswordProtectedError# Make a request with the password. Access to the file/folder will be stored in cookies- async with self.request_limiter:- json_resp: dict = await self.client.post_data(- self.DOMAIN,- self.FOLDER_PASSWORD_API_URL,- data={- "folderPassword": password,- "folderId": node_id,- "submitme": 1,- },- )+ json_resp: dict[str, Any] = await self.request_json(+ self.FOLDER_PASSWORD_API_URL,+ method="POST",+ data={+ "folderPassword": password,+ "folderId": node_id,+ "submitme": 1,+ },+ )if not json_resp.get("success"):raise PasswordProtectedError(message="Incorrect password")diff --git a/cyberdrop_dl/crawlers/archivebate.py b/cyberdrop_dl/crawlers/archivebate.pyindex b558f1a7d..53df85361 100644--- a/cyberdrop_dl/crawlers/archivebate.py+++ b/cyberdrop_dl/crawlers/archivebate.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.mixdrop import MixDropCrawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError@@ -11,8 +9,6 @@ from cyberdrop_dl.utils import css, open_graphfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.crawlers.crawler import SupportedDomains, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -37,9 +33,7 @@ class ArchiveBateCrawler(MixDropCrawler):FOLDER_DOMAIN: ClassVar[str] = "ArchiveBate"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLNEXT_PAGE_SELECTOR = _SELECTORS.NEXT_PAGE-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(4, 1)+ _RATE_LIMIT = 4, 1async def fetch(self, scrape_item: ScrapeItem) -> None:if "watch" in scrape_item.url.parts:@@ -68,8 +62,7 @@ class ArchiveBateCrawler(MixDropCrawler):self.manager.progress_manager.download_progress.add_previously_completed()return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)if "This video has been deleted" in soup.text:raise ScrapeError(410)@@ -88,8 +81,7 @@ class ArchiveBateCrawler(MixDropCrawler):if await self.check_complete_from_referer(mixdrop_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, mixdrop_url)+ soup = await self.request_soup(mixdrop_url)link = self.create_download_link(soup)filename, ext = self.get_filename_and_ext(link.name)diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.pyindex 5c71f6d83..cd7aeaf6b 100644--- a/cyberdrop_dl/crawlers/ashemaletube.py+++ b/cyberdrop_dl/crawlers/ashemaletube.py@@ -4,8 +4,6 @@ import jsonfrom enum import StrEnumfrom typing import TYPE_CHECKING, ClassVar, NamedTuple-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError@@ -81,9 +79,7 @@ class AShemaleTubeCrawler(Crawler):FOLDER_DOMAIN: ClassVar[str] = "aShemaleTube"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLNEXT_PAGE_SELECTOR: ClassVar[str] = _SELECTORS.NEXT_PAGE-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(3, 10)+ _RATE_LIMIT = 3, 10async def fetch(self, scrape_item: ScrapeItem) -> None:if any(p in scrape_item.url.parts for p in ("creators", "profiles", "pornstars", "model")):@@ -108,7 +104,7 @@ class AShemaleTubeCrawler(Crawler):async def gallery(self, scrape_item: ScrapeItem) -> None:async for soup in self.web_pager(scrape_item.url, cffi=True):for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.GALLERY_ALBUM):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:@@ -132,7 +128,7 @@ class AShemaleTubeCrawler(Crawler):else:scrape_item.setup_as_album(collection_title)for _, new_scrape_item in self.iter_children(scrape_item, soup, MEDIA_SELECTOR_MAP[collection_type]):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))def create_collection_title(self, soup: BeautifulSoup, collection_type: CollectionType) -> str:title_elem = soup.select_one(TITLE_SELECTOR_MAP[collection_type])@@ -147,8 +143,7 @@ class AShemaleTubeCrawler(Crawler):async def image(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item.url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url, impersonate=True)img_item = soup.select_one(_SELECTORS.IMAGE_ITEM)if not img_item:raise ScrapeError(404)@@ -173,8 +168,7 @@ class AShemaleTubeCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url, impersonate=True)if soup.select_one(_SELECTORS.LOGIN_REQUIRED):raise ScrapeError(401)diff --git a/cyberdrop_dl/crawlers/beeg.py b/cyberdrop_dl/crawlers/beeg.pyindex fcd6dd4c0..c7bf20b7e 100644--- a/cyberdrop_dl/crawlers/beeg.py+++ b/cyberdrop_dl/crawlers/beeg.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Any, ClassVar, NamedTuple-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -32,24 +30,20 @@ class BeegComCrawler(Crawler):}DOMAIN: ClassVar[str] = "beeg.com"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL+ _RATE_LIMIT = 4, 1async def fetch(self, scrape_item: ScrapeItem) -> None:if video_id := get_video_id(scrape_item.url):return await self.video(scrape_item, video_id)raise ValueError- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(4, 1)-async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:canonical_url = PRIMARY_URL / video_idif await self.check_complete_from_referer(canonical_url):returnscrape_item.url = canonical_url- async with self.request_limiter:- json_resp: dict[str, Any] = await self.client.get_json(self.DOMAIN, JSON_URL / video_id)-+ json_resp: dict[str, Any] = await self.request_json(JSON_URL / video_id)facts: dict[str, Any] = min(json_resp["fc_facts"], key=lambda x: int(x["id"]))file: dict[str, Any] = json_resp["file"]title: str = next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"]diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.pyindex dad94b496..30dc4bc04 100644--- a/cyberdrop_dl/crawlers/box_dot_com.py+++ b/cyberdrop_dl/crawlers/box_dot_com.py@@ -16,8 +16,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -75,8 +73,7 @@ class BoxDotComCrawler(Crawler):if "file" in scrape_item.url.parts and await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)if "file or folder link has been removed" in soup.text:raise ScrapeError(410)@@ -97,7 +94,7 @@ class BoxDotComCrawler(Crawler):_, file_id = file_key.rsplit("f_", 1)canonical_url = get_canonical_url(shared_name, file_id)scrape_item.url = canonical_url- self.manager.task_group.create_task(self.run(scrape_item))+ self.create_task(self.run(scrape_item))returnshared_folder = SharedFolder(**shared_folder_data)diff --git a/cyberdrop_dl/crawlers/bunkrr_albums_io.py b/cyberdrop_dl/crawlers/bunkrr_albums_io.pyindex 9bd10b712..e61ea6e6a 100644--- a/cyberdrop_dl/crawlers/bunkrr_albums_io.py+++ b/cyberdrop_dl/crawlers/bunkrr_albums_io.py@@ -37,17 +37,16 @@ class BunkrAlbumsIOCrawler(Crawler):search_query = scrape_item.url.query["search"]title = self.create_title(search_query)scrape_item.setup_as_profile(title)- async for soup in self.web_pager(scrape_item):+ async for soup in self._pager(scrape_item):for _, new_scrape_item in self.iter_children(scrape_item, soup, ALBUM_SELECTOR):self.handle_external_links(new_scrape_item)- async def web_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:+ async def _pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:"""Generator of website pages."""page_url = scrape_item.urlwhile True:current_page_number = int(page_url.query.get("page") or 1)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, page_url)+ soup = await self.request_soup(page_url)next_page = soup.select(self.NEXT_PAGE_SELECTOR)yield soupif not next_page:diff --git a/cyberdrop_dl/crawlers/buzzheavier.py b/cyberdrop_dl/crawlers/buzzheavier.pyindex c3858dab5..aa0b2229d 100644--- a/cyberdrop_dl/crawlers/buzzheavier.py+++ b/cyberdrop_dl/crawlers/buzzheavier.py@@ -27,12 +27,16 @@ class BuzzHeavierCrawler(Crawler):returnurl = scrape_item.url / "download"- headers = {"HX-Current-URL": str(scrape_item.url), "HX-Request": "true"}- async with self.request_limiter:- response = await self.client._get_head(self.DOMAIN, url, headers=headers)-- assert response.content_disposition- assert response.content_disposition.filename- link = self.parse_url(response.headers["hx-redirect"])- filename, ext = self.get_filename_and_ext(response.content_disposition.filename, assume_ext=".zip")+ async with self.request(+ url,+ method="HEAD",+ headers={+ "HX-Current-URL": str(scrape_item.url),+ "HX-Request": "true",+ },+ ) as resp:+ filename: str = resp.filename++ link = self.parse_url(resp.headers["hx-redirect"])+ filename, ext = self.get_filename_and_ext(filename, assume_ext=".zip")await self.handle_file(scrape_item.url, scrape_item, filename, ext, debrid_link=link)diff --git a/cyberdrop_dl/crawlers/camwhores_dot_tv.py b/cyberdrop_dl/crawlers/camwhores_dot_tv.pyindex a559fae16..ef03d8387 100644--- a/cyberdrop_dl/crawlers/camwhores_dot_tv.py+++ b/cyberdrop_dl/crawlers/camwhores_dot_tv.py@@ -40,7 +40,7 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler):return video._replace(title=open_graph.title(soup))def parse_url(- self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = False+ self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool | None = None) -> AbsoluteHttpURL:return super().parse_url(link_str, relative_to, trim=False)@@ -57,8 +57,7 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler):async def iter_videos(self, scrape_item: ScrapeItem, video_category: str = "") -> None:url = scrape_item.url / video_category if video_category else scrape_item.urlawait super().iter_videos(scrape_item, video_category)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, url)+ soup = await self.request_soup(url)last_page = int(css.get_text(soup.select(LAST_PAGE_SELECTOR)[-1]))# TODO: Porntrex also uses KVS. Make the KVS crawler handle it by defaultdiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 86db0caf1..7c7006775 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -270,14 +270,16 @@ class Crawler(ABC):try:yield task_idexcept ValueError:- await self.raise_e(scrape_item, ScrapeError("Unknown URL path"))+ self.raise_exc(scrape_item, ScrapeError("Unknown URL path"))except MaxChildrenError as e:- await self.raise_e(scrape_item, e)+ self.raise_exc(scrape_item, e)finally:pass@error_handling_wrapper- def raise_e(self, scrape_item: ScrapeItem, exc: type[Exception] | Exception) -> None:+ def raise_exc(self, scrape_item: ScrapeItem, exc: type[Exception] | Exception | str) -> None:+ if isinstance(exc, str):+ exc = ScrapeError(exc)raise exc@final@@ -427,7 +429,7 @@ class Crawler(ABC):def handle_external_links(self, scrape_item: ScrapeItem) -> None:"""Maps external links to the scraper class."""scrape_item.reset()- self.manager.task_group.create_task(self.manager.scrape_mapper.filter_and_send_to_crawler(scrape_item))+ self.create_task(self.manager.scrape_mapper.filter_and_send_to_crawler(scrape_item))# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~@@ -660,6 +662,24 @@ class Crawler(ABC):async with self.request(url, method="HEAD") as head:return head.location or url+ @final+ @error_handling_wrapper+ async def follow_redirect_w_get(self, scrape_item: ScrapeItem) -> None:+ async with self.request(scrape_item.url) as resp:+ if resp.url == scrape_item.url:+ raise ScrapeError(422)+ scrape_item.url = resp.url+ self.create_task(self.run(scrape_item))++ @final+ @error_handling_wrapper+ async def follow_redirect_w_head(self, scrape_item: ScrapeItem) -> None:+ location = await self._get_redirect_url(scrape_item.url)+ if location == scrape_item.url:+ raise ScrapeError(422)+ scrape_item.url = location+ self.create_task(self.run(scrape_item))+@staticmethoddef register_cache_filter(url: URL, filter_fn: Callable[[AnyResponse], bool] | Callable[[AnyResponse], Awaitable[bool]]diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.pyindex 59825576e..81c91d77e 100644--- a/cyberdrop_dl/crawlers/cyberdrop.py+++ b/cyberdrop_dl/crawlers/cyberdrop.py@@ -1,9 +1,7 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar--from aiolimiter import AsyncLimiter-from bs4 import BeautifulSoup+import asyncio+from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -12,8 +10,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemAPI_ENTRYPOINT = AbsoluteHttpURL("https://api.cyberdrop.me/api/")@@ -40,9 +36,6 @@ class CyberdropCrawler(Crawler):_RATE_LIMIT: ClassVar[tuple[float, float]] = 5, 1_DOWNLOAD_SLOTS: ClassVar[int | None] = 1- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(1, 2)-async def fetch(self, scrape_item: ScrapeItem) -> None:if "a" in scrape_item.url.parts:return await self.album(scrape_item)@@ -53,8 +46,7 @@ class CyberdropCrawler(Crawler):scrape_item.url = scrape_item.url.with_query("nojs")album_id = scrape_item.url.parts[2]- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)try:title: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_TITLE)@@ -68,36 +60,32 @@ class CyberdropCrawler(Crawler):scrape_item.possible_datetime = self.parse_date(date_tags[-1].text, "%d.%m.%Y")for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUM_ITEM):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:- scrape_item.url = await self.get_stream_link(scrape_item.url)+ scrape_item.url = await self._get_stream_link(scrape_item.url)if await self.check_complete_from_referer(scrape_item):returnfile_id = scrape_item.url.name- async with self.request_limiter:- api_url = API_ENTRYPOINT / "file" / "info" / file_id- json_resp = await self.client.get_json(self.DOMAIN, api_url)-- filename, ext = self.get_filename_and_ext(json_resp["name"])+ file_info: tuple[dict[str, Any], dict[str, Any]] = await asyncio.gather(+ self.request_json(API_ENTRYPOINT / "file" / "info" / file_id),+ self.request_json(API_ENTRYPOINT / "file" / "auth" / file_id),+ )- async with self.request_limiter:- api_url = API_ENTRYPOINT / "file" / "auth" / file_id- json_resp = await self.client.get_json(self.DOMAIN, api_url)-- link = self.parse_url(json_resp["url"])+ filename, ext = self.get_filename_and_ext(file_info[0]["name"])+ link = self.parse_url(file_info[1]["url"])await self.handle_file(link, scrape_item, filename, ext)- async def get_stream_link(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:- """Gets the stream link for a given URL.-- NOTE: This makes a request to get the final URL (if necessary). Calling function must use `@error_handling_wrapper`"""+ async def _get_stream_link(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ """Gets the stream link for a given URL."""if any(part in url.parts for part in ("a", "f")):return url+if url.host.count(".") > 1 or "e" in url.parts:return PRIMARY_URL / "f" / url.name- response, _ = await self.client._get_response_and_soup(self.DOMAIN, url)- return AbsoluteHttpURL(response.url)++ async with self.request(url) as resp:+ return resp.urldiff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.pyindex 661b56fa7..66bb46857 100644--- a/cyberdrop_dl/crawlers/dirtyship.py+++ b/cyberdrop_dl/crawlers/dirtyship.py@@ -66,8 +66,7 @@ class DirtyShipCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):returnif not scrape_item.url.suffix == ".jpg":- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)url = self.parse_url(next(css.get_attr(a, "href") for a in soup.select(_SELECTORS.SINGLE_PHOTO) if "full" in a.get_text()))@@ -118,12 +117,11 @@ class DirtyShipCrawler(Crawler):scrape_item.setup_as_album(title)for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.PLAYLIST_ITEM):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)title: str = css.select_one_get_text(soup, "title")title = title.split(" - DirtyShip")[0]diff --git a/cyberdrop_dl/crawlers/discourse/_discourse.py b/cyberdrop_dl/crawlers/discourse/_discourse.pyindex 7493038e4..b318f0c5e 100644--- a/cyberdrop_dl/crawlers/discourse/_discourse.py+++ b/cyberdrop_dl/crawlers/discourse/_discourse.py@@ -65,8 +65,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):async def make_request(self, model_cls: type[_ModelT], path: str, params: dict[str, Any] | None = None) -> _ModelT:api_url = self.PRIMARY_URL.joinpath(path)- async with self.request_limiter:- json_text = await self.client.get_text(self.DOMAIN, api_url, request_params={"params": params})+ json_text = await self.request_text(api_url, params=params)return model_cls.model_validate_json(json_text, by_alias=True, by_name=True)@error_handling_wrapper@@ -87,13 +86,13 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):await self.handle_link(scrape_item, self.parse_url(topic.image_url))await self.process_posts(scrape_item, topic)- async def forum(self, scrape_item: ScrapeItem) -> None:+ async def forum(self, scrape_item: ScrapeItem, /) -> None:raise NotImplementedErrordef parse_thread(self, *args, **kwargs) -> None:raise NotImplementedError- async def thread(self, scrape_item: ScrapeItem, thread: Topic) -> None:+ async def thread(self, scrape_item: ScrapeItem, /, thread: Topic) -> None:await self.topic(scrape_item, thread)@error_handling_wrapper@@ -127,7 +126,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):if topic.init_post_number != 1 and self.scrape_single_forum_post:return- async def post(self, scrape_item: ScrapeItem, post: AvailablePost) -> None:+ async def post(self, scrape_item: ScrapeItem, /, post: AvailablePost) -> None:title = self.create_separate_post_title(post.title, str(post.id), post.created_at)scrape_item.setup_as_post(title)for link in self.extract_links(post):@@ -149,7 +148,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):return unique(iter_links())def parse_url(- self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True+ self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool | None = None) -> AbsoluteHttpURL:return _clean_url(super().parse_url(link_str, relative_to, trim=trim))diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.pyindex 427926a6b..fbb0f992d 100644--- a/cyberdrop_dl/crawlers/doodstream.py+++ b/cyberdrop_dl/crawlers/doodstream.py@@ -57,18 +57,13 @@ class DoodStreamCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- response, soup = await self.client._get_response_and_soup_cffi(self.DOMAIN, scrape_item.url)+ async with self.request(scrape_item.url, impersonate=True) as resp:+ actual_host = resp.url.host+ soup = await resp.soup()- host = self.parse_url(response.url).host- assert host- del response-- title: str = css.select_one_get_text(soup, "title")- title = title.split("- DoodStream")[0].strip()-- file_id = get_file_id(soup)- debrid_link = await self.get_download_url(host, soup)+ title = css.page_title(soup, "DoodStream")+ file_id = _get_file_id(soup)+ debrid_link = await self.get_download_url(actual_host, soup)filename, ext = self.get_filename_and_ext(f"{file_id}.mp4")custom_filename = self.create_custom_filename(title, ext, file_id=file_id)scrape_item.url = canonical_url@@ -77,25 +72,23 @@ class DoodStreamCrawler(Crawler):)async def get_download_url(self, host: str, soup: BeautifulSoup) -> AbsoluteHttpURL:- md5_path = get_md5_path(soup)+ md5_path = _get_md5_path(soup)api_url = API_MD5_ENTRYPOINT / md5_pathtoken = api_url.name- async with self.request_limiter:- new_soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, api_url.with_host(host))- text = new_soup.get_text(strip=True)+ text = await self.request_text(api_url.with_host(host), impersonate=True)random_padding = "".join(random.choice(TOKEN_CHARS) for _ in range(10))expire = int(datetime.now(UTC).timestamp() * 1000)download_url = self.parse_url(text + random_padding)return download_url.with_query(token=token, expiry=expire)-def get_md5_path(soup: BeautifulSoup) -> str:+def _get_md5_path(soup: BeautifulSoup) -> str:js_text = css.select_one_get_text(soup, _SELECTORS.MD5_JS)return get_text_between(js_text, "/pass_md5/", "'")-def get_file_id(soup: BeautifulSoup) -> str:+def _get_file_id(soup: BeautifulSoup) -> str:js_text = css.select_one_get_text(soup, _SELECTORS.FILE_ID_JS)_, file_id, _ = js_text.split("'file_id'")[-1].split("'", 2)return file_iddiff --git a/cyberdrop_dl/crawlers/dropbox.py b/cyberdrop_dl/crawlers/dropbox.pyindex 4740e161a..683c5bba0 100644--- a/cyberdrop_dl/crawlers/dropbox.py+++ b/cyberdrop_dl/crawlers/dropbox.py@@ -46,7 +46,7 @@ class DropboxCrawler(Crawler):preview_filename = scrape_item.url.query.get("preview")match scrape_item.url.parts[1:]:case ["s" | "sh", _, *_]:- return await self.follow_redirect(scrape_item)+ return await self.follow_redirect_w_head(scrape_item)case ["scl", "fo", token1, token2]:url = PRIMARY_URL / "scl/fo" / token1 / token2if preview_filename:@@ -78,32 +78,15 @@ class DropboxCrawler(Crawler):raise ScrapeError(422, message="Folders download is not enabled")scrape_item.url = item.view_url- filename = item.filename or await self.get_content_disposition_name(item.download_url)- if not filename:- raise ScrapeError(422)+ if item.filename:+ filename = item.filename+ else:+ async with self.request(item.download_url) as resp:+ filename = resp.filename+filename, ext = self.get_filename_and_ext(filename)await self.handle_file(item.url, scrape_item, filename, ext, debrid_link=item.download_url)- @error_handling_wrapper- async def follow_redirect(self, scrape_item: ScrapeItem) -> None:- scrape_item.url = await self.get_redict_url(scrape_item.url)- await self.fetch(scrape_item)-- async def get_content_disposition_name(self, url: AbsoluteHttpURL) -> str | None:- url = await self.get_redict_url(url)- async with self.request_limiter:- response = await self.client._get_head(self.DOMAIN, url)- if response.content_disposition:- return response.content_disposition.filename-- async def get_redict_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:- async with self.request_limiter:- headers = await self.client.get_head(self.DOMAIN, url)- location = headers.get("location")- if not location:- raise ScrapeError(400)- return self.parse_url(location)-@dataclass(frozen=True, slots=True)class DropboxItem:diff --git a/cyberdrop_dl/crawlers/e621.py b/cyberdrop_dl/crawlers/e621.pyindex 1161f0791..b35b12f6d 100644--- a/cyberdrop_dl/crawlers/e621.py+++ b/cyberdrop_dl/crawlers/e621.py@@ -3,8 +3,6 @@ from __future__ import annotationsimport itertoolsfrom typing import TYPE_CHECKING, Any, ClassVar-from aiolimiter import AsyncLimiter-from cyberdrop_dl import __version__from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -28,10 +26,10 @@ class E621Crawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "e621.net"FOLDER_DOMAIN: ClassVar[str] = "E621"+ _RATE_LIMIT = 2, 1def __post_init__(self) -> None:self.headers = {"User-Agent": f"CyberDrop-DL/{__version__} (by B05FDD249DF29ED3)"}- self.request_limiter = AsyncLimiter(2, 1)async def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.query.get("tags"):@@ -42,15 +40,13 @@ class E621Crawler(Crawler):return await self.pool(scrape_item)raise ValueError- async def paginator(self, scrape_item: ScrapeItem) -> AsyncGenerator[list[dict[str, Any]]]:+ async def _pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[list[dict[str, Any]]]:"""Generator for album pages."""initial_page = int(scrape_item.url.query.get("page", 1))url = PRIMARY_URL / "posts.json"for page in itertools.count(initial_page):url = url.with_query(tags=scrape_item.url.query["tags"], page=page)- async with self.request_limiter:- json_resp: dict = await self.client.get_json(self.DOMAIN, url, self.headers)-+ json_resp: dict[str, Any] = await self.request_json(url, headers=self.headers)posts: list[dict] = json_resp.get("posts", [])if not posts:break@@ -63,7 +59,7 @@ class E621Crawler(Crawler):title = self.create_title(tags.replace("+", " "))scrape_item.setup_as_album(title)- async for posts in self.paginator(scrape_item):+ async for posts in self._pager(scrape_item):for post in posts:try:file_url = post["file"]["url"]@@ -81,9 +77,7 @@ class E621Crawler(Crawler):"""Fetches posts from an e621 pool."""pool_id = scrape_item.url.nameurl = PRIMARY_URL / f"pools/{pool_id}.json"- async with self.request_limiter:- json_resp: dict = await self.client.get_json(self.DOMAIN, url, self.headers)-+ json_resp: dict[str, Any] = await self.request_json(url, headers=self.headers)posts = json_resp.get("post_ids", [])title: str = json_resp.get("name", "Unknown Pool").replace("_", " ")scrape_item.setup_as_album(title)@@ -91,7 +85,7 @@ class E621Crawler(Crawler):for post_id in posts:url = PRIMARY_URL / f"posts/{post_id}"new_scrape_item = scrape_item.create_child(url)- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))scrape_item.add_children()@error_handling_wrapper@@ -99,11 +93,9 @@ class E621Crawler(Crawler):"""Fetches a single post by extracting the ID from the URL."""post_id = scrape_item.url.nameurl = PRIMARY_URL / f"posts/{post_id}.json"- async with self.request_limiter:- json_resp: dict = await self.client.get_json(self.DOMAIN, url, self.headers)-+ json_resp: dict[str, Any] = await self.request_json(url, headers=self.headers)try:- file_url = json_resp["post"]["file"]["url"]+ file_url: str = json_resp["post"]["file"]["url"]except KeyError:raise ScrapeError(422) from Nonediff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.pyindex 11fedc80c..7a56d5f02 100644--- a/cyberdrop_dl/crawlers/efukt.py+++ b/cyberdrop_dl/crawlers/efukt.py@@ -11,8 +11,6 @@ from cyberdrop_dl.utils.dates import to_timestampfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemPRIMARY_URL = AbsoluteHttpURL("https://efukt.com")@@ -60,15 +58,14 @@ class EfuktCrawler(Crawler):scrape_item.setup_as_album(self.create_title(f"{title} [series]"))for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEO_THUMBS):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def video_image_or_gif(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)date_str = css.select_one_get_text(soup, _SELECTORS.DATE).split(" ", 1)[-1]datetime = self._parse_date(date_str, "%m/%d/%y")diff --git a/cyberdrop_dl/crawlers/ehentai.py b/cyberdrop_dl/crawlers/ehentai.pyindex 4365b5fa2..97defd350 100644--- a/cyberdrop_dl/crawlers/ehentai.py+++ b/cyberdrop_dl/crawlers/ehentai.py@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -53,27 +51,25 @@ class EHentaiCrawler(Crawler):await self.set_no_warnings(scrape_item)title: str = ""+ gallery_id = scrape_item.url.parts[2]scrape_item.url = scrape_item.url.with_query(None)async for soup in self.web_pager(scrape_item.url):if not title:title = self.create_title(css.select_one_get_text(soup, _SELECTORS.TITLE))date_str: str = css.select_one_get_text(soup, _SELECTORS.DATE)- gallery_id = scrape_item.url.parts[2]title = self.create_title(title, gallery_id)scrape_item.setup_as_album(title, album_id=gallery_id)scrape_item.possible_datetime = self.parse_date(date_str)for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUM_IMAGES):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)-+ soup = await self.request_soup(scrape_item.url)link_str: str = css.select_one_get_attr(soup, _SELECTORS.IMAGE, "src")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)@@ -84,6 +80,5 @@ class EHentaiCrawler(Crawler):async def set_no_warnings(self, scrape_item: ScrapeItem) -> None:"""Sets the no warnings cookie."""url = scrape_item.url.update_query(nw="session")- async with self.request_limiter:- await self.client.get_soup(self.DOMAIN, url)- self._warnings_set = True+ async with self.request(url):+ self._warnings_set = Truediff --git a/cyberdrop_dl/crawlers/eightmuses.py b/cyberdrop_dl/crawlers/eightmuses.pyindex a00026e72..d7ae35939 100644--- a/cyberdrop_dl/crawlers/eightmuses.py+++ b/cyberdrop_dl/crawlers/eightmuses.py@@ -33,8 +33,7 @@ class EightMusesCrawler(Crawler):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)album_id = scrape_item.album_idif not album_id:diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.pyindex a291712a8..8d9a322fa 100644--- a/cyberdrop_dl/crawlers/eporner.py+++ b/cyberdrop_dl/crawlers/eporner.py@@ -116,7 +116,7 @@ class EpornerCrawler(Crawler):url = canonical_url / partasync for soup in self.web_pager(url):for _, new_scrape_item in self.iter_children(scrape_item, soup, selector, new_title_part=name):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def playlist(self, scrape_item: ScrapeItem, from_profile: bool = False) -> None:@@ -131,7 +131,7 @@ class EpornerCrawler(Crawler):scrape_item.setup_as_album(title)for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEO):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def gallery(self, scrape_item: ScrapeItem) -> None:@@ -156,8 +156,7 @@ class EpornerCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)scrape_item.url = canonical_urllink_str = css.select_one_get_attr(soup, _SELECTORS.PHOTO, "href")@@ -172,8 +171,7 @@ class EpornerCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)soup_str = soup.textif "File has been removed due to copyright owner request" in soup_str:diff --git a/cyberdrop_dl/crawlers/erome.py b/cyberdrop_dl/crawlers/erome.pyindex e48dce10d..0829c694c 100644--- a/cyberdrop_dl/crawlers/erome.py+++ b/cyberdrop_dl/crawlers/erome.py@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -48,15 +46,14 @@ class EromeCrawler(Crawler):scrape_item.setup_as_profile(title)for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUM):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:album_id = scrape_item.url.parts[2]results = await self.get_album_results(album_id)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)title_portion = css.select_one_get_text(soup, "title").rsplit(" - Porn")[0].strip()if not title_portion:diff --git a/cyberdrop_dl/crawlers/fapello.py b/cyberdrop_dl/crawlers/fapello.pyindex 5c9b7bf4d..752fca3d4 100644--- a/cyberdrop_dl/crawlers/fapello.py+++ b/cyberdrop_dl/crawlers/fapello.py@@ -2,16 +2,12 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -30,9 +26,7 @@ class FapelloCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLNEXT_PAGE_SELECTOR: ClassVar[str] = 'div[id="next_page"] a'DOMAIN: ClassVar[str] = "fapello"-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(5, 1)+ _RATE_LIMIT = 5, 1async def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.name:@@ -62,8 +56,7 @@ class FapelloCrawler(Crawler):@error_handling_wrapperasync def post(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)content = css.select_one(soup, POST_CONTENT_SELECTOR)for _, link in self.iter_tags(content, "img, source"):diff --git a/cyberdrop_dl/crawlers/fileditch.py b/cyberdrop_dl/crawlers/fileditch.pyindex 8d0181533..4d86f5dad 100644--- a/cyberdrop_dl/crawlers/fileditch.py+++ b/cyberdrop_dl/crawlers/fileditch.py@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemDOWNLOAD_SELECTOR = "a[class*='download-button']"@@ -32,8 +30,7 @@ class FileditchCrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_str: str = css.select_one_get_attr(soup, DOWNLOAD_SELECTOR, "href")link = self.parse_url(link_str)if link.path == HOMEPAGE_CATCHALL_FILE:diff --git a/cyberdrop_dl/crawlers/files_vc.py b/cyberdrop_dl/crawlers/files_vc.pyindex 571591a12..dae3b0d6c 100644--- a/cyberdrop_dl/crawlers/files_vc.py+++ b/cyberdrop_dl/crawlers/files_vc.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Any, ClassVar-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils.utilities import error_handling_wrapper@@ -22,9 +20,7 @@ class FilesVcCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "files.vc"FOLDER_DOMAIN: ClassVar[str] = "FilesVC"-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(1, 1)+ _RATE_LIMIT = 1, 1async def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.path == "/d/dl" and scrape_item.url.query.get("hash"):@@ -39,8 +35,7 @@ class FilesVcCrawler(Crawler):hash = scrape_item.url.query["hash"]api_url = API_ENTRYPOINT.joinpath("info").with_query(hash=hash)- async with self.request_limiter:- json_resp: dict[str, Any] = await self.client.get_json(self.DOMAIN, api_url)+ json_resp: dict[str, Any] = await self.request_json(api_url)filename, ext = self.get_filename_and_ext(json_resp["filename"], assume_ext=".zip")scrape_item.possible_datetime = self.parse_date(json_resp["upload_time"])diff --git a/cyberdrop_dl/crawlers/fourchan.py b/cyberdrop_dl/crawlers/fourchan.pyindex 30049d066..5b18b723d 100644--- a/cyberdrop_dl/crawlers/fourchan.py+++ b/cyberdrop_dl/crawlers/fourchan.py@@ -2,7 +2,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar, NotRequired, TypedDict, cast-from aiolimiter import AsyncLimiterfrom bs4 import BeautifulSoupfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths@@ -44,9 +43,7 @@ class FourChanCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "4chan"_DOWNLOAD_SLOTS: ClassVar[int | None] = 1-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(3, 10)+ _RATE_LIMIT = 3, 10async def fetch(self, scrape_item: ScrapeItem) -> None:if "thread" in scrape_item.url.parts:@@ -59,8 +56,7 @@ class FourChanCrawler(Crawler):async def thread(self, scrape_item: ScrapeItem) -> None:board, _, thread_id = scrape_item.url.parts[1:4]api_url = API_ENTRYPOINT / board / f"thread/{thread_id}.json"- async with self.request_limiter:- response: dict[str, list[Post]] = await self.client.get_json(self.DOMAIN, api_url, cache_disabled=True)+ response: dict[str, list[Post]] = await self.request_json(api_url, cache_disabled=True)if not response:raise ScrapeError(404)@@ -94,13 +90,11 @@ class FourChanCrawler(Crawler):async def board(self, scrape_item: ScrapeItem) -> None:board: str = scrape_item.url.parts[-1]api_url = API_ENTRYPOINT / board / "threads.json"- async with self.request_limiter:- threads: list[ThreadList] = await self.client.get_json(self.DOMAIN, api_url, cache_disabled=True)-+ threads: list[ThreadList] = await self.request_json(api_url, cache_disabled=True)scrape_item.setup_as_forum("")for page in threads:for thread in page["threads"]:url = PRIMARY_URL / board / f"thread/{thread['no']}"new_scrape_item = scrape_item.create_child(url)- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))scrape_item.add_children()diff --git a/cyberdrop_dl/crawlers/generic.py b/cyberdrop_dl/crawlers/generic.pyindex 040b4382b..d696e05f6 100644--- a/cyberdrop_dl/crawlers/generic.py+++ b/cyberdrop_dl/crawlers/generic.py@@ -63,9 +63,10 @@ class GenericCrawler(Crawler):@error_handling_wrapper@log_unsupported_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:- content_type = await self.get_content_type(scrape_item.url)- if "html" in content_type:- return await self.try_video_from_soup(scrape_item)+ async with self.request(scrape_item.url) as resp:+ content_type = resp.content_type+ if "html" in content_type:+ return await self.try_video_from_soup(scrape_item, await resp.soup())filename, ext = guess_filename_and_ext(scrape_item.url, content_type)if not ext:@@ -75,18 +76,7 @@ class GenericCrawler(Crawler):filename, _ = self.get_filename_and_ext(fullname.name)await self.handle_file(scrape_item.url, scrape_item, filename, ext)- async def get_content_type(self, url: AbsoluteHttpURL) -> str:- async with self.request_limiter:- headers = await self.client.get_head(self.DOMAIN, url)- content_type: str = headers.get("Content-Type", "")- if not content_type:- raise ScrapeError(422)- return content_type.lower()-- async def try_video_from_soup(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)-+ async def try_video_from_soup(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> None:try:title = css.select_one_get_text(soup, "title").rsplit(" - ", 1)[0].rsplit("|", 1)[0]link_str: str = css.select_one_get_attr(soup, VIDEO_SELECTOR, "src")diff --git a/cyberdrop_dl/crawlers/girlsreleased.py b/cyberdrop_dl/crawlers/girlsreleased.pyindex d23215fea..6246aa786 100644--- a/cyberdrop_dl/crawlers/girlsreleased.py+++ b/cyberdrop_dl/crawlers/girlsreleased.py@@ -67,14 +67,10 @@ class GirlsReleasedCrawler(Crawler):case _:raise ValueError- async def _api_request(self, api_url: AbsoluteHttpURL) -> dict[str, Any]:- async with self.request_limiter:- return await self.client.get_json(self.DOMAIN, api_url)-@error_handling_wrapperasync def set(self, scrape_item: ScrapeItem, set_id: str) -> None:api_url = self.PRIMARY_URL / "api/0.1/set" / set_id- set_data = (await self._api_request(api_url))["set"]+ set_data = (await self.request_json(api_url))["set"]self._handle_set(scrape_item, _parse_set(set_data))@error_handling_wrapper@@ -85,7 +81,7 @@ class GirlsReleasedCrawler(Crawler):for page in itertools.count(0):api_url = base_api_url / str(page)- sets: list[dict[str, Any]] = (await self._api_request(api_url))["sets"]+ sets: list[dict[str, Any]] = (await self.request_json(api_url))["sets"]for set_data in sets:set_ = _parse_set(set_data)diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.pyindex 9027cb0f6..9f96f13c8 100644--- a/cyberdrop_dl/crawlers/gofile.py+++ b/cyberdrop_dl/crawlers/gofile.py@@ -7,8 +7,6 @@ from datetime import UTC, datetime, timedeltafrom hashlib import sha256from typing import TYPE_CHECKING, ClassVar, Literal, NotRequired, TypedDict, cast-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import FILE_HOST_ALBUM, AbsoluteHttpURL, ScrapeItemfrom cyberdrop_dl.exceptions import DownloadError, PasswordProtectedError, ScrapeError@@ -18,7 +16,7 @@ if TYPE_CHECKING:from collections.abc import Mapping-WT_REGEX = re.compile(r'appdata\.wt\s=\s"([^"]+)"')+_FIND_WT = re.compile(r'appdata\.wt\s=\s"([^"]+)"').searchAPI_ENTRYPOINT = AbsoluteHttpURL("https://api.gofile.io")GLOBAL_JS_URL = AbsoluteHttpURL("https://gofile.io/dist/js/global.js")PRIMARY_URL = AbsoluteHttpURL("https://gofile.io")@@ -87,13 +85,12 @@ class GoFileCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "gofile"FOLDER_DOMAIN: ClassVar[str] = "GoFile"- _RATE_LIMIT: ClassVar[tuple[float, float]] = 100, 60+ _RATE_LIMIT: ClassVar[tuple[float, float]] = 4, 10def __post_init__(self) -> None:self.api_key = self.manager.config_manager.authentication_data.gofile.api_keyself.website_token = self.manager.cache_manager.get("gofile_website_token")self.headers: dict[str, str] = {}- self.request_limiter = AsyncLimiter(4, 6)self._website_token_date = datetime.now(UTC) - timedelta(days=7)async def async_startup(self) -> None:@@ -117,8 +114,7 @@ class GoFileCrawler(Crawler):api_url = api_url.update_query(password=sha256_password)try:- async with self.request_limiter:- json_resp: ApiAlbumResponse = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)+ json_resp: ApiAlbumResponse = await self.request_json(api_url, headers=self.headers)except DownloadError as e:if e.status != http.HTTPStatus.UNAUTHORIZED:@@ -126,8 +122,7 @@ class GoFileCrawler(Crawler):async with self.startup_lock:await self.get_website_token(update=True)api_url = api_url.update_query(wt=self.website_token)- async with self.request_limiter:- json_resp = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)+ json_resp = await self.request_json(api_url, headers=self.headers)album = get_album_data(json_resp)if is_single_not_nested_file(scrape_item, album):@@ -190,7 +185,7 @@ class GoFileCrawler(Crawler):for url in subfolders:subfolder = scrape_item.create_child(url)- self.manager.task_group.create_task(self.run(subfolder))+ self.create_task(self.run(subfolder))tasks = []for url in unavailable:@@ -216,8 +211,7 @@ class GoFileCrawler(Crawler):async def _get_new_api_key(self) -> str:api_url = API_ENTRYPOINT / "accounts"- async with self.request_limiter:- json_resp = await self.client.post_data(self.DOMAIN, api_url, data={})+ json_resp = await self.request_json(api_url, method="POST", data={})if json_resp["status"] != "ok":raise ScrapeError(401, "Couldn't generate GoFile API token", origin=api_url)@@ -235,11 +229,11 @@ class GoFileCrawler(Crawler):await self._update_website_token()async def _update_website_token(self) -> None:- async with self.request_limiter:- text = await self.client.get_text(self.DOMAIN, GLOBAL_JS_URL)- match = WT_REGEX.search(str(text))+ text = await self.request_text(GLOBAL_JS_URL)+ match = _FIND_WT(text)if not match:raise ScrapeError(401, "Couldn't generate GoFile websiteToken", origin=GLOBAL_JS_URL)+self.website_token = match.group(1)self.manager.cache_manager.save("gofile_website_token", self.website_token)self._website_token_date = datetime.now(UTC)diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.pyindex eeed3f962..e13c46c89 100644--- a/cyberdrop_dl/crawlers/google_drive.py+++ b/cyberdrop_dl/crawlers/google_drive.py@@ -33,7 +33,7 @@from __future__ import annotationsimport asyncio-from typing import TYPE_CHECKING, ClassVar, cast+from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -121,8 +121,7 @@ class GoogleDriveCrawler(Crawler):@error_handling_wrapperasync def folder(self, scrape_item: ScrapeItem, folder_id: str) -> None:embeded_folder_url = (self.PRIMARY_URL / "embeddedfolderview").with_query(id=folder_id)- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, embeded_folder_url)+ soup = await self.request_soup(embeded_folder_url)folder_name = css.select_one_get_text(soup, "title")title = self.create_title(folder_name, folder_id)@@ -191,14 +190,10 @@ class GoogleDriveCrawler(Crawler):# Use POST request to bypass "file is too large to scan. Would you still like to download this file" warningmethod = "GET" if export_url.host == _DOCS_URL.host else "POST"- # TODO: This request bypasses the config limiter. Use the new request method when PR #1251 is merged- async with self.request_limiter, self.client._session.request(method, export_url) as resp:- if not resp.ok or "html" in resp.content_type:- await self.client.client_manager.check_http_status(resp)+ async with self.request(export_url, method=method) as resp:+ assert resp.ok and "html" not in resp.content_type- direct_url = cast("AbsoluteHttpURL", resp.url)- filename: str = resp.content_disposition.filename # type: ignore[reportAssignmentType,reportOptionalMemberAccess]- return direct_url, filename+ return resp.url, resp.filenamedef _get_proper_doc_format(doc: str, format: str | None) -> str:diff --git a/cyberdrop_dl/crawlers/hitomi_la.py b/cyberdrop_dl/crawlers/hitomi_la.pyindex 985c3f221..20369edd3 100644--- a/cyberdrop_dl/crawlers/hitomi_la.py+++ b/cyberdrop_dl/crawlers/hitomi_la.py@@ -8,8 +8,6 @@ from collections import defaultdictfrom datetime import datetime, timedeltafrom typing import TYPE_CHECKING, ClassVar, NamedTuple, Required, TypedDict-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError@@ -91,7 +89,6 @@ class HitomiLaCrawler(Crawler):def __post_init__(self) -> None:self._semaphore = asyncio.Semaphore(3)- self.request_limiter = AsyncLimiter(1, 1)self.headers = {"Referer": str(PRIMARY_URL), "Origin": str(PRIMARY_URL)}self._servers: Servers | None = None@@ -117,10 +114,10 @@ class HitomiLaCrawler(Crawler):nozomi_url = LTN_SERVER / colletion_type / f"{name}-{language}.nozomi"scrape_item.setup_as_profile(title)- async with self.request_limiter:- response, _ = await self.client._get(self.DOMAIN, nozomi_url, self.headers)+ async with self.request(nozomi_url, headers=self.headers) as response:+ content = await response.read()- for gallery_id in decode_nozomi_response(await response.read()):+ for gallery_id in decode_nozomi_response(content):new_scrape_item = scrape_item.create_child(PRIMARY_URL / f"galleries/{gallery_id}.html")# await immediately to prevent overwhelming the downloader# there could be thousands of galleries in the result@@ -145,9 +142,9 @@ class HitomiLaCrawler(Crawler):# This is partial implementation. Only parses tagged words, ex `female:dark_skin`# Free form query searches are ignoredfor nozomi_search_args in (parse_query_word(word) for word in search_query.split(" ") if ":" in word):- async with self.request_limiter:- response, _ = await self.client._get(self.DOMAIN, nozomi_search_args.url, self.headers)- yield set(decode_nozomi_response(await response.read()))+ async with self.request(nozomi_search_args.url, headers=self.headers) as response:+ content = await response.read()+ yield set(decode_nozomi_response(content))@error_handling_wrapperasync def gallery(self, scrape_item: ScrapeItem) -> None:@@ -165,8 +162,7 @@ class HitomiLaCrawler(Crawler):async def get_gallery(self, gallery_id: str) -> Gallery:gallery_url = LTN_SERVER / f"galleries/{gallery_id}.js"- async with self.request_limiter:- js_text = await self.client.get_text(self.DOMAIN, gallery_url, self.headers)+ js_text = await self.request_text(gallery_url, headers=self.headers)return json.loads(js_text.split("=", 1)[-1])async def get_servers(self) -> Servers:@@ -177,9 +173,8 @@ class HitomiLaCrawler(Crawler):async def _get_servers(self) -> Servers:# https://ltn.gold-usergeneratedcontent.net/gg.js- async with self.request_limiter:- js_text = await self.client.get_text(self.DOMAIN, LTN_SERVER / "gg.js", cache_disabled=True)+ js_text = await self.request_text(LTN_SERVER / "gg.js", cache_disabled=True)root, num, default_num = [match_int_or_none(pattern, js_text) for pattern in (_REGEX.ROOT, _REGEX.NUM, _REGEX.DEFAULT_NUM)]diff --git a/cyberdrop_dl/crawlers/hotpic.py b/cyberdrop_dl/crawlers/hotpic.pyindex 5591139c4..73b185241 100644--- a/cyberdrop_dl/crawlers/hotpic.py+++ b/cyberdrop_dl/crawlers/hotpic.py@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -46,8 +44,7 @@ class HotPicCrawler(Crawler):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)album_id = scrape_item.url.parts[2]title = self.create_title(css.select_one_get_text(soup, "title").rsplit(" - ")[0], scrape_item.album_id)@@ -61,8 +58,7 @@ class HotPicCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_str = css.select_one_get_attr(soup, _SELECTORS.IMAGE_OR_VIDEO, "src")link = self.parse_url(link_str)diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.pyindex 1f55efcc6..f1d5b489c 100644--- a/cyberdrop_dl/crawlers/imagebam.py+++ b/cyberdrop_dl/crawlers/imagebam.py@@ -49,7 +49,7 @@ class ImageBamCrawler(Crawler):# this is to update the URL in the UI# TODO: Add URL convertion methods as a pre_check step, before assigning them a task_id# Several crawlers do something like this- self.manager.task_group.create_task(self.run(scrape_item))+ self.create_task(self.run(scrape_item))returnmatch scrape_item.url.parts[1:]:@@ -65,8 +65,7 @@ class ImageBamCrawler(Crawler):@error_handling_wrapperasync def view(self, scrape_item: ScrapeItem) -> None:# view URLs can be either a gallery or a single image.- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)if soup.select_one(_SELECTORS.IS_GALLERY):return await self.gallery(scrape_item, soup)await self.image(scrape_item, soup)@@ -74,8 +73,7 @@ class ImageBamCrawler(Crawler):@error_handling_wrapperasync def gallery(self, scrape_item: ScrapeItem, soup: BeautifulSoup | None = None) -> None:if not soup:- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)gallery_name = css.select_one_get_text(soup, _SELECTORS.GALLERY_TITLE)gallery_id = scrape_item.url.name@@ -86,13 +84,12 @@ class ImageBamCrawler(Crawler):def process_gallery_images(soup: BeautifulSoup) -> None:for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.THUMBNAILS):if not self.check_album_results(new_scrape_item.url, results):- self.manager.task_group.create_task(self.image(new_scrape_item))+ self.create_task(self.image(new_scrape_item))process_gallery_images(soup)while next_page := css.select_one_get_attr_or_none(soup, _SELECTORS.NEXT_PAGE, "href"):- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, self.parse_url(next_page))+ soup = await self.request_soup(self.parse_url(next_page))process_gallery_images(soup)@error_handling_wrapper@@ -101,8 +98,7 @@ class ImageBamCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)image_tag = css.select_one(soup, _SELECTORS.IMAGE)if not scrape_item.album_id and (gallery_info := soup.select_one(_SELECTORS.GALLERY_INFO)):diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.pyindex 47cf146e8..b28bfba0b 100644--- a/cyberdrop_dl/crawlers/imgbb.py+++ b/cyberdrop_dl/crawlers/imgbb.py@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemPRIMARY_URL = AbsoluteHttpURL("https://ibb.co")@@ -58,20 +56,19 @@ class ImgBBCrawler(Crawler):first_page = self.parse_url(first_page_str)for _, sub_album in self.iter_children(scrape_item, soup, ALBUM_PAGE_SELECTOR):- self.manager.task_group.create_task(self.run(sub_album))+ self.create_task(self.run(sub_album))assert first_pageasync for soup in self.web_pager(first_page):for _, image in self.iter_children(scrape_item, soup, IMAGE_PAGE_SELECTOR):- self.manager.task_group.create_task(self.run(image))+ self.create_task(self.run(image))@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")link = self.parse_url(link_str)diff --git a/cyberdrop_dl/crawlers/imgbox.py b/cyberdrop_dl/crawlers/imgbox.pyindex ddc72dff1..214acb82a 100644--- a/cyberdrop_dl/crawlers/imgbox.py+++ b/cyberdrop_dl/crawlers/imgbox.py@@ -1,6 +1,6 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import FILE_HOST_ALBUM, AbsoluteHttpURL, ScrapeItem@@ -8,9 +8,6 @@ from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper-if TYPE_CHECKING:- from bs4 import BeautifulSoup-PRIMARY_URL = AbsoluteHttpURL("https://imgbox.com")IMAGES_SELECTOR = "div#gallery-view-content img"IMAGE_SELECTOR = "img[id=img]"@@ -37,8 +34,7 @@ class ImgBoxCrawler(Crawler):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)if "The specified gallery could not be found" in soup.text:raise ScrapeError(404)@@ -62,8 +58,7 @@ class ImgBoxCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")link = self.parse_url(link_str)diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.pyindex 8fa3c388e..df16a5998 100644--- a/cyberdrop_dl/crawlers/imgur.py+++ b/cyberdrop_dl/crawlers/imgur.py@@ -54,20 +54,17 @@ class ImgurCrawler(Crawler):album_id = scrape_item.url.parts[-1]title: str = ""- async with self.request_limiter:- await self.check_imgur_credits(scrape_item)- api_url = API_ENTRYPOINT / "album" / album_id- json_resp: dict[str, dict] = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)+ await self.check_imgur_credits(scrape_item)+ api_url = API_ENTRYPOINT / "album" / album_id+ json_resp: dict[str, dict[str, Any]] = await self.request_json(api_url, headers=self.headers)- title_part = json_resp["data"].get("title", album_id)- title = self.create_title(title_part, album_id)- scrape_item.setup_as_album(title, album_id=album_id)+ title: str = json_resp["data"].get("title", album_id)+ scrape_item.setup_as_album(self.create_title(title, album_id), album_id=album_id)- async with self.request_limiter:- api_url = API_ENTRYPOINT / "album" / album_id / "images"- json_resp = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)+ api_url = API_ENTRYPOINT / "album" / album_id / "images"+ images: dict[str, list[dict[str, Any]]] = await self.request_json(api_url, headers=self.headers)- for image in json_resp["data"]:+ for image in images["data"]:await self.process_image(scrape_item, image)@error_handling_wrapper@@ -77,10 +74,9 @@ class ImgurCrawler(Crawler):raise LoginError(msg)image_id = scrape_item.url.parts[-1]- async with self.request_limiter:- await self.check_imgur_credits(scrape_item)- api_url = API_ENTRYPOINT / "image" / image_id- json_resp = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)+ await self.check_imgur_credits(scrape_item)+ api_url = API_ENTRYPOINT / "image" / image_id+ json_resp: dict[str, dict[str, Any]] = await self.request_json(api_url, headers=self.headers)await self.process_image(scrape_item, json_resp["data"])@@ -104,8 +100,7 @@ class ImgurCrawler(Crawler):async def check_imgur_credits(self, _=None) -> None:"""Checks the remaining credits."""- credits_url = API_ENTRYPOINT / "credits"- json_resp = await self.client.get_json(self.DOMAIN, credits_url, headers=self.headers)+ json_resp = await self.request_json(API_ENTRYPOINT / "credits", headers=self.headers)self.imgur_client_remaining = json_resp["data"]["ClientRemaining"]if self.imgur_client_remaining < 100:raise ScrapeError(429, "Imgur API rate limit reached")diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.pyindex 66d4fd7d5..2bf18a362 100644--- a/cyberdrop_dl/crawlers/imx_to.py+++ b/cyberdrop_dl/crawlers/imx_to.py@@ -38,9 +38,7 @@ class ImxToCrawler(Crawler):returndata = {"imgContinue": "Continue+to+image+...+"}- async with self.request_limiter:- soup = await self.client.post_data_get_soup(self.DOMAIN, scrape_item.url, data=data)-+ soup = await self.request_soup(scrape_item.url, method="POST", data=data)link_str: str = css.select_one_get_attr(soup, IMG_SELECTOR, "src")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name, assume_ext=".jpg")diff --git a/cyberdrop_dl/crawlers/incestflix.py b/cyberdrop_dl/crawlers/incestflix.pyindex 7ebda098a..600b0886c 100644--- a/cyberdrop_dl/crawlers/incestflix.py+++ b/cyberdrop_dl/crawlers/incestflix.py@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -44,8 +42,7 @@ class IncestflixCrawler(Crawler):async def video(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)title: str = css.select_one_get_text(soup, _SELECTORS.TITLE)link_str = css.select_one_get_attr(soup, _SELECTORS.VIDEO, "src")url = self.parse_url(link_str)@@ -61,4 +58,4 @@ class IncestflixCrawler(Crawler):scrape_item.setup_as_album(title)title_created = Truefor _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEO_THUMBS):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))diff --git a/cyberdrop_dl/crawlers/influencer_bitches.py b/cyberdrop_dl/crawlers/influencer_bitches.pyindex 3948d329d..fa8e48bb1 100644--- a/cyberdrop_dl/crawlers/influencer_bitches.py+++ b/cyberdrop_dl/crawlers/influencer_bitches.py@@ -38,8 +38,7 @@ class InfluencerBitchesCrawler(Crawler):@error_handling_wrapperasync def model(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)title_tag = soup.select_one(_SELECTORS.TITLE) or soup.select_one(_SELECTORS.ALTERNATIVE_TITLE)assert title_tagtitle: str = title_tag.get_text(strip=True).replace("leaks", "").strip()diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.pyindex 498b2d3a0..172799656 100644--- a/cyberdrop_dl/crawlers/jpg5.py+++ b/cyberdrop_dl/crawlers/jpg5.py@@ -1,7 +1,5 @@from __future__ import annotations-from aiolimiter import AsyncLimiter-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom ._chevereto import CheveretoCrawler@@ -28,8 +26,7 @@ class JPG5Crawler(CheveretoCrawler):"jpg5.su",)- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(1, 1)+ _RATE_LIMIT = 1, 1@classmethoddef transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:diff --git a/cyberdrop_dl/crawlers/leakedzone.py b/cyberdrop_dl/crawlers/leakedzone.pyindex be75fa98b..6dff01633 100644--- a/cyberdrop_dl/crawlers/leakedzone.py+++ b/cyberdrop_dl/crawlers/leakedzone.py@@ -5,8 +5,6 @@ import dataclassesimport itertoolsfrom typing import TYPE_CHECKING, Any, ClassVar-from aiolimiter import AsyncLimiter-from cyberdrop_dl.compat import IntEnumfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -64,6 +62,7 @@ class LeakedZoneCrawler(Crawler):FOLDER_DOMAIN: ClassVar[str] = "LeakedZone"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLIMAGES_CDN: ClassVar[AbsoluteHttpURL] = IMAGES_CDN+ _RATE_LIMIT = 3, 10async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:@@ -74,9 +73,6 @@ class LeakedZoneCrawler(Crawler):case _:raise ValueError- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(3, 10)-@classmethoddef get_encoded_video_url(cls, soup: BeautifulSoup) -> str:js_text = css.select_one_get_text(soup, _SELECTORS.JW_PLAYER)@@ -84,15 +80,15 @@ class LeakedZoneCrawler(Crawler):@error_handling_wrapperasync def model(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)model_name: str = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME_FROM_PROFILE)scrape_item.setup_as_profile(self.create_title(model_name))headers = {"X-Requested-With": "XMLHttpRequest"}for page in itertools.count(1):- async with self.request_limiter:- posts = await self.client.get_json(self.DOMAIN, scrape_item.url.with_query(page=page), headers=headers)+ posts: list[dict[str, Any]] = await self.request_json(+ scrape_item.url.with_query(page=page), headers=headers+ )# We may be able to omit the last request by just checking the number of posts# Seens to always return 48 postsif not posts:@@ -111,8 +107,7 @@ class LeakedZoneCrawler(Crawler):if await self.check_complete_from_referer(scrape_item.url):return- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)model_name = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME)scrape_item.setup_as_album(self.create_title(model_name))diff --git a/cyberdrop_dl/crawlers/luscious.py b/cyberdrop_dl/crawlers/luscious.pyindex 78c097c26..e3014a485 100644--- a/cyberdrop_dl/crawlers/luscious.py+++ b/cyberdrop_dl/crawlers/luscious.py@@ -39,7 +39,7 @@ class LusciousCrawler(Crawler):return await self.search(scrape_item)await self.album(scrape_item)- async def create_graphql_query(self, operation: str, scrape_item: ScrapeItem, page: int = 1) -> str:+ def create_graphql_query(self, operation: str, scrape_item: ScrapeItem, page: int = 1) -> str:"""Creates a graphql query."""assert operation in GRAPHQL_QUERIESquery = scrape_item.url.query@@ -71,16 +71,14 @@ class LusciousCrawler(Crawler):results = await self.get_album_results(album_id)title: str = ""query_name = "AlbumGet"- async with self.request_limiter:- query = await self.create_graphql_query(query_name, scrape_item)- json_resp = await self.make_post_request(query_name, query)-+ query = self.create_graphql_query(query_name, scrape_item)+ json_resp = await self._api_request(query_name, query)title = self.create_title(json_resp["data"]["album"]["get"]["title"], album_id)scrape_item.setup_as_album(title, album_id=album_id)- async for json_resp in self.paginator(scrape_item, is_album=True):- for item in json_resp["data"]["picture"]["list"]["items"]:- link = self.parse_url(item["url_to_original"])+ async for albums in self._pager(scrape_item, is_album=True):+ for album in albums:+ link = self.parse_url(album["url_to_original"])if not self.check_album_results(link, results):filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)@@ -94,28 +92,31 @@ class LusciousCrawler(Crawler):if not query:raise ScrapeError(400, "No search query provided")- async for json_data in self.paginator(scrape_item):- for item in json_data["data"]["album"]["list"]["items"]:- album_url = self.parse_url(item["url"])+ async for results in self._pager(scrape_item):+ for album in results:+ album_url = self.parse_url(album["url"])new_scrape_item = scrape_item.create_child(url=album_url)await self.album(new_scrape_item)- async def paginator(self, scrape_item: ScrapeItem, is_album: bool = False) -> AsyncGenerator[dict]:+ async def _pager(self, scrape_item: ScrapeItem, is_album: bool = False) -> AsyncGenerator[list[dict[str, Any]]]:"""Generator for album pages."""initial_page = int(scrape_item.url.query.get("page", 1))query_name = "PictureListInsideAlbum" if is_album else "AlbumListWithPeek"data_name = "picture" if is_album else "album"for page in itertools.count(initial_page):- query: str = await self.create_graphql_query(query_name, scrape_item, page)- json_resp = await self.make_post_request(query_name, query)- yield json_resp- if not json_resp["data"][data_name]["list"]["info"]["has_next_page"]:+ query = self.create_graphql_query(query_name, scrape_item, page)+ results: dict[str, Any] = (await self._api_request(query_name, query))["data"][data_name]["list"]+ yield results["items"]+ if not results["info"]["has_next_page"]:break- async def make_post_request(self, query_name: str, query: str) -> dict:+ async def _api_request(self, query_name: str, query: str) -> dict[str, Any]:api_url = GRAPHQL_URL.with_query(operationName=query_name)- headers = {"Content-Type": "application/json"}- async with self.request_limiter:- json_resp = await self.client.post_data(self.DOMAIN, api_url, data=query, headers=headers)+ json_resp = await self.request_json(+ api_url,+ method="POST",+ data=query,+ headers={"Content-Type": "application/json"},+ )log_debug(json_resp)return json_respdiff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 0391e8b30..215974037 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -3,7 +3,6 @@ from __future__ import annotationsimport itertoolsfrom typing import TYPE_CHECKING, Any, ClassVar-from aiolimiter import AsyncLimiterfrom mediafire import MediaFireApi, apifrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths@@ -13,8 +12,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemDOWNLOAD_BUTTON_SELECTOR = "a[id=downloadButton]"@@ -29,10 +26,10 @@ class MediaFireCrawler(Crawler):}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "mediafire"+ _RATE_LIMIT = 5, 1def __post_init__(self) -> None:self.api = MediaFireApi()- self.request_limiter = AsyncLimiter(5, 1)async def fetch(self, scrape_item: ScrapeItem) -> None:if "folder" in scrape_item.url.parts:@@ -62,7 +59,7 @@ class MediaFireCrawler(Crawler):date = self.parse_date(file["created"])link = self.parse_url(file["links"]["normal_download"])new_scrape_item = scrape_item.create_child(link, new_title_part=title, possible_datetime=date)- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))scrape_item.add_children()if not folder_contents["folder_content"]["more_chunks"] == "yes":@@ -73,8 +70,7 @@ class MediaFireCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_tag = soup.select_one(DOWNLOAD_BUTTON_SELECTOR)if not link_tag:diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex 9bba1210a..080ba05eb 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -7,7 +7,6 @@ It calls checks_complete_by_referer several times even if no request is going tofrom __future__ import annotations-import itertoolsfrom typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, castfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id@@ -60,7 +59,7 @@ class MegaNzCrawler(Crawler):return self.manager.auth_config.meganz.password or Nonedef _init_downloader(self) -> mega.MegaDownloader:- self.downloader = dl = mega.MegaDownloader(self.manager, self.DOMAIN)+ self.downloader = dl = mega.MegaDownloader(self.manager, self.DOMAIN) # type: ignore[reportIncompatibleVariableOverride]dl.startup()return dl@@ -171,16 +170,15 @@ class MegaNzCrawler(Crawler):) -> None:folder_id, shared_key = scrape_item.url.name, scrape_item.url.fragment- def exclude_node(fs_entry: tuple[Path, mega.Node]):- node = fs_entry[1]- if node["t"] != mega.NodeType.FILE:- return True- if single_file_id and node["h"] != single_file_id:- return True- return False+ def filter_files():+ for path, node in filesystem.items():+ if node["t"] != mega.NodeType.FILE:+ continue+ if single_file_id and node["h"] != single_file_id:+ continue+ yield path, cast("mega.File", node)- for path, node in itertools.filterfalse(exclude_node, filesystem.items()):- file = cast("mega.File", node)+ for path, file in filter_files():file_id = file["h"]file_fragment = f"{shared_key}/file/{file_id}"canonical_url = scrape_item.url.with_fragment(file_fragment)diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.pyindex b0ca902f8..dfa380aa2 100644--- a/cyberdrop_dl/crawlers/missav.py+++ b/cyberdrop_dl/crawlers/missav.py@@ -35,8 +35,7 @@ class MissAVCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url, impersonate=True)title, date_str = open_graph.title(soup), open_graph.get("video_release_date", soup)if dvd_code_tag := soup.select_one(DVD_CODE_SELECTOR):diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.pyindex c29aeb0e7..f8ea481d0 100644--- a/cyberdrop_dl/crawlers/mixdrop.py+++ b/cyberdrop_dl/crawlers/mixdrop.py@@ -51,13 +51,11 @@ class MixDropCrawler(Crawler):returnscrape_item.url = embed_url- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, video_url)+ soup = await self.request_soup(video_url)title = css.select_one_get_text(soup, _SELECTOR.FILE_NAME)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, embed_url)+ soup = await self.request_soup(embed_url)link = self.create_download_link(soup)filename, ext = self.get_filename_and_ext(link.name)diff --git a/cyberdrop_dl/crawlers/motherless.py b/cyberdrop_dl/crawlers/motherless.pyindex 0861dcc5c..b8e592171 100644--- a/cyberdrop_dl/crawlers/motherless.py+++ b/cyberdrop_dl/crawlers/motherless.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError@@ -43,9 +41,7 @@ class MotherlessCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLNEXT_PAGE_SELECTOR: ClassVar[str] = "div.pagination_link > a[rel=next]"DOMAIN: ClassVar[str] = "motherless"-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(2, 1)+ _RATE_LIMIT = 2, 1async def fetch(self, scrape_item: ScrapeItem, collection_id: str = "") -> None:parts = scrape_item.url.parts@@ -87,13 +83,13 @@ class MotherlessCrawler(Crawler):async for soup in self.web_pager(images_url):check_soup(soup)for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part="Images"):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))if is_homepage or "videos" in scrape_item.url.parts:async for soup in self.web_pager(videos_url):check_soup(soup)for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part="Videos"):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def collection(self, scrape_item: ScrapeItem) -> None:@@ -134,7 +130,7 @@ class MotherlessCrawler(Crawler):scrape_item.setup_as_album(title, album_id=collection_id)for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part=name):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def media(self, scrape_item: ScrapeItem) -> None:@@ -144,8 +140,7 @@ class MotherlessCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)check_soup(soup)media_info = self.process_media_soup(scrape_item, soup)diff --git a/cyberdrop_dl/crawlers/nhentai.py b/cyberdrop_dl/crawlers/nhentai.pyindex f5466e785..b08e070fc 100644--- a/cyberdrop_dl/crawlers/nhentai.py+++ b/cyberdrop_dl/crawlers/nhentai.py@@ -3,8 +3,6 @@ from __future__ import annotationsimport randomfrom typing import TYPE_CHECKING, Any, ClassVar-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import LoginError@@ -41,9 +39,7 @@ class NHentaiCrawler(Crawler):NEXT_PAGE_SELECTOR = "a.next"DOMAIN = "nhentai.net"FOLDER_DOMAIN = "nHentai"-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(4, 1)+ _RATE_LIMIT = 4, 1async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:@@ -80,14 +76,11 @@ class NHentaiCrawler(Crawler):async def gallery(self, scrape_item: ScrapeItem) -> None:gallery_id = scrape_item.url.nameapi_url = self.PRIMARY_URL / "api/gallery" / gallery_id-- async with self.request_limiter:- json_resp: dict[str, Any] = await self.client.request_json_cffi(self.DOMAIN, api_url)+ json_resp: dict[str, Any] = await self.request_json(api_url, impersonate=True)titles: dict[str, str] = json_resp["title"]title: str = titles.get("english") or titles.get("japanese") or titles["pretty"]- title = self.create_title(title, gallery_id)- scrape_item.setup_as_album(title, album_id=gallery_id)+ scrape_item.setup_as_album(self.create_title(title, gallery_id), album_id=gallery_id)scrape_item.possible_datetime = json_resp["upload_date"]padding = max(3, len(str(json_resp["num_pages"])))diff --git a/cyberdrop_dl/crawlers/noodle_magazine.py b/cyberdrop_dl/crawlers/noodle_magazine.pyindex 94b67a29b..52960b611 100644--- a/cyberdrop_dl/crawlers/noodle_magazine.py+++ b/cyberdrop_dl/crawlers/noodle_magazine.py@@ -4,9 +4,6 @@ import itertoolsimport jsonfrom typing import TYPE_CHECKING, Any, ClassVar, NamedTuple-from aiolimiter import AsyncLimiter-from bs4 import BeautifulSoup-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError@@ -14,8 +11,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -44,9 +39,7 @@ class NoodleMagazineCrawler(Crawler):DOMAIN: ClassVar[str] = "noodlemagazine"FOLDER_DOMAIN: ClassVar[str] = "NoodleMagazine"_DOWNLOAD_SLOTS: ClassVar[int | None] = 2-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(1, 3)+ _RATE_LIMIT = 1, 3async def fetch(self, scrape_item: ScrapeItem) -> None:if "video" in scrape_item.url.parts:@@ -63,8 +56,7 @@ class NoodleMagazineCrawler(Crawler):for page in itertools.count(1, init_page):n_videos = 0page_url = scrape_item.url.with_query(p=page)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, page_url)+ soup = await self.request_soup(page_url, impersonate=True)if not title:search_string: str = css.select_one_get_text(soup, SEARCH_STRING_SELECTOR)@@ -76,7 +68,7 @@ class NoodleMagazineCrawler(Crawler):if new_scrape_item.url not in seen_urls:seen_urls.add(new_scrape_item.url)n_videos += 1- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))if n_videos < 24:break@@ -85,8 +77,7 @@ class NoodleMagazineCrawler(Crawler):async def video(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item.url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url, impersonate=True)metadata_text = css.select_one(soup, METADATA_SELECTOR).get_text()metadata = json.loads(metadata_text.strip())diff --git a/cyberdrop_dl/crawlers/nudostartv.py b/cyberdrop_dl/crawlers/nudostartv.pyindex 329f06786..b084f019f 100644--- a/cyberdrop_dl/crawlers/nudostartv.py+++ b/cyberdrop_dl/crawlers/nudostartv.py@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemIMAGE_SELECTOR = "div[class=block-video] a img"@@ -47,15 +45,14 @@ class NudoStarTVCrawler(Crawler):raise ScrapeError(404)for _, new_scrape_item in self.iter_children(scrape_item, soup, CONTENT_SELECTOR):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)diff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.pyindex c6ee32b6a..fde93f958 100644--- a/cyberdrop_dl/crawlers/odnoklassniki.py+++ b/cyberdrop_dl/crawlers/odnoklassniki.py@@ -71,8 +71,7 @@ class OdnoklassnikiCrawler(Crawler):@error_handling_wrapperasync def channel(self, scrape_item: ScrapeItem, channel_str: str):- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url, _HEADERS)+ soup = await self.request_soup(scrape_item.url, headers=_HEADERS)channel_id = channel_str.removeprefix("c")gwt_hash = get_text_between(css.select_one_get_text(soup, Selector.CHANNEL_HASH), 'gwtHash:"', '",')@@ -107,26 +106,24 @@ class OdnoklassnikiCrawler(Crawler):breakpage += 1- async with self.request_limiter:- resp = await self.client._post_data(- self.DOMAIN,- page_url,- _HEADERS,- data={- "fetch": "false",- "st.page": page,- "st.lastelem": last_element_id,- "gwt.requested": gwt_hash,- },- )- last_element_id = resp.headers.get("lastelem")- content = await resp.text()+ async with self.request(+ page_url,+ method="POST",+ headers=_HEADERS,+ data={+ "fetch": "false",+ "st.page": page,+ "st.lastelem": last_element_id,+ "gwt.requested": gwt_hash,+ },+ ) as resp:+ last_element_id = resp.headers.get("lastelem")+ content = await resp.text()@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str):mobile_url = AbsoluteHttpURL(f"https://m.ok.ru/video/{video_id}")- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, mobile_url, _MOBILE_HEADERS)+ soup = await self.request_soup(mobile_url, headers=_MOBILE_HEADERS)_check_video_is_available(soup)metadata: dict[str, Any] = json.loads(Selector.FLASHVARS(soup))["flashvars"]["metadata"]diff --git a/cyberdrop_dl/crawlers/omegascans.py b/cyberdrop_dl/crawlers/omegascans.pyindex 938b25fb5..3b48e7d4d 100644--- a/cyberdrop_dl/crawlers/omegascans.py+++ b/cyberdrop_dl/crawlers/omegascans.py@@ -1,7 +1,7 @@from __future__ import annotationsimport itertools-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -10,8 +10,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -43,8 +41,7 @@ class OmegaScansCrawler(Crawler):@error_handling_wrapperasync def series(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)series_id = Nonejs_script = soup.select_one(JS_SELECTOR)@@ -55,15 +52,14 @@ class OmegaScansCrawler(Crawler):scrape_item.setup_as_album("", album_id=series_id)# TODO: Add title# title: str = ""- for page in itertools.count(1):- api_url = API_ENTRYPOINT.with_query(page=page, perPage=30, series_id=series_id)- async with self.request_limiter:- json_resp = await self.client.get_json(self.DOMAIN, api_url)+ api_url = API_ENTRYPOINT.with_query(series_id=series_id, perPage=30)+ for page in itertools.count(1):+ json_resp: dict[str, Any] = await self.request_json(api_url.update_query(page=page))for chapter in json_resp["data"]:chapter_url = scrape_item.url / chapter["chapter_slug"]new_scrape_item = scrape_item.create_child(chapter_url)- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))scrape_item.add_children()if json_resp["meta"]["current_page"] == json_resp["meta"]["last_page"]:@@ -71,8 +67,7 @@ class OmegaScansCrawler(Crawler):@error_handling_wrapperasync def chapter(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)if "This chapter is premium" in soup.get_text():raise ScrapeError(401, "This chapter is premium")diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.pyindex 3c2a70730..da9be4401 100644--- a/cyberdrop_dl/crawlers/onedrive.py+++ b/cyberdrop_dl/crawlers/onedrive.py@@ -4,7 +4,6 @@from __future__ import annotations-import jsonfrom dataclasses import dataclassfrom datetime import UTC, datetime, timedeltafrom functools import partial@@ -132,13 +131,8 @@ class OneDriveCrawler(Crawler):@error_handling_wrapperasync def share_link(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- headers = await self.client.get_head(self.DOMAIN, scrape_item.url)- location = headers.get("location")- if not location:- raise ScrapeError(400)og_share_link = scrape_item.url- scrape_item.url = self.parse_url(location)+ scrape_item.url = await self._get_redirect_url(scrape_item.url)await self.link_with_credentials(scrape_item, og_share_link)@error_handling_wrapper@@ -200,7 +194,7 @@ class OneDriveCrawler(Crawler):for access_details in subfolders:api_url = create_api_url(access_details)new_scrape_item = scrape_item.create_child(api_url)- self.manager.task_group.create_task(self.process_access_details(new_scrape_item, access_details))+ self.create_task(self.process_access_details(new_scrape_item, access_details))scrape_item.add_children()@error_handling_wrapper@@ -213,20 +207,22 @@ class OneDriveCrawler(Crawler):await self.handle_file(file.url, scrape_item, filename, ext, debrid_link=file.download_url)async def make_api_request(self, api_url: AbsoluteHttpURL) -> dict[str, Any]:- headers = {"Content-Type": "application/json"} | self.auth_headers- async with self.request_limiter:- json_resp: dict = await self.client.get_json(self.DOMAIN, api_url, headers=headers)-- return json_resp+ return await self.request_json(+ api_url,+ headers={+ "Content-Type": "application/json",+ }+ | self.auth_headers,+ )@error_handling_wrapperasync def get_badger_token(self, badger_url: AbsoluteHttpURL = BADGER_URL) -> None:- new_headers = {"Content-Type": "application/json", "AppId": APP_ID}- data = {"appId": APP_UUID}- data_json = json.dumps(data)- async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.DOMAIN, badger_url, headers=new_headers, data=data_json)-+ json_resp: dict[str, Any] = await self.request_json(+ badger_url,+ method="POST",+ headers={"Content-Type": "application/json", "AppId": APP_ID},+ json={"appId": APP_UUID},+ )badger_token: str = json_resp["token"]badger_token_expires: str = json_resp["expiryTimeUtc"]self.auth_headers = {"Prefer": "autoredeem", "Authorization": f"Badger {badger_token}"}diff --git a/cyberdrop_dl/crawlers/pimpandhost.py b/cyberdrop_dl/crawlers/pimpandhost.pyindex b74e2259e..156d880ec 100644--- a/cyberdrop_dl/crawlers/pimpandhost.py+++ b/cyberdrop_dl/crawlers/pimpandhost.py@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemALBUM_TITLE_SELECTOR = "span[class=author-header__album-name]"@@ -47,12 +45,11 @@ class PimpAndHostCrawler(Crawler):scrape_item.possible_datetime = self.parse_date(css.get_attr(date_tag, "title"), DATE_FORMAT)for _, new_scrape_item in self.iter_children(scrape_item, soup, FILES_SELECTOR):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "data-src")link = self.parse_url(link_str)diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.pyindex e23eb96e7..519cbb6ad 100644--- a/cyberdrop_dl/crawlers/pixeldrain.py+++ b/cyberdrop_dl/crawlers/pixeldrain.py@@ -1,7 +1,7 @@from __future__ import annotationsimport json-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -11,8 +11,6 @@ from cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemAPI_ENTRYPOINT = AbsoluteHttpURL("https://pixeldrain.com/api/")@@ -38,9 +36,8 @@ class PixelDrainCrawler(Crawler):album_id = scrape_item.url.parts[2]results = await self.get_album_results(album_id)- async with self.request_limiter:- api_url = API_ENTRYPOINT / "list" / scrape_item.url.parts[-1]- json_resp = await self.client.get_json(self.DOMAIN, api_url)+ api_url = API_ENTRYPOINT / "list" / scrape_item.url.parts[-1]+ json_resp = await self.request_json(api_url)log_debug(json_resp)title = self.create_title(json_resp["title"], album_id)@@ -70,10 +67,9 @@ class PixelDrainCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return+ api_url = API_ENTRYPOINT / "file" / scrape_item.url.name / "info"try:- async with self.request_limiter:- api_url = API_ENTRYPOINT / "file" / scrape_item.url.name / "info"- json_resp = await self.client.get_json(self.DOMAIN, api_url)+ json_resp: dict[str, Any] = await self.request_json(api_url)except DownloadError as e:if e.status != 404:raise@@ -81,7 +77,7 @@ class PixelDrainCrawler(Crawler):link = self.create_download_link(json_resp["id"])scrape_item.possible_datetime = self.parse_date(json_resp["date_upload"])- filename = json_resp["name"]+ filename: str = json_resp["name"]mime_type: str = json_resp["mime_type"]try:filename, ext = self.get_filename_and_ext(filename)@@ -99,9 +95,8 @@ class PixelDrainCrawler(Crawler):await self.handle_file(link, scrape_item, filename, ext)async def text(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- api_url = API_ENTRYPOINT / "file" / scrape_item.url.parts[-1]- text: str = await self.client.get_text(self.DOMAIN, api_url)+ api_url = API_ENTRYPOINT / "file" / scrape_item.url.parts[-1]+ text = await self.request_text(api_url)for line in text.splitlines():link = self.parse_url(line)@@ -110,8 +105,7 @@ class PixelDrainCrawler(Crawler):scrape_item.add_children()async def filesystem(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)og_props = open_graph.parse(soup)filename = og_props.titlediff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.pyindex a6d9214fa..14cbe3642 100644--- a/cyberdrop_dl/crawlers/pixhost.py+++ b/cyberdrop_dl/crawlers/pixhost.py@@ -41,15 +41,14 @@ class PixHostCrawler(Crawler):match scrape_item.url.parts[1:]:case ["gallery", gallery_id]:return await self.gallery(scrape_item, gallery_id)- case ["show", _]:+ case ["show", _, *_]:return await self.image(scrape_item)case _:raise ValueError@error_handling_wrapperasync def gallery(self, scrape_item: ScrapeItem, album_id: str) -> None:- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)title = css.select_one_get_text(soup, _SELECTORS.GALLERY_TITLE)title = self.create_title(title, album_id)@@ -69,8 +68,7 @@ class PixHostCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_str = css.select_one_get_attr(soup, _SELECTORS.IMAGE, "src")link = self.parse_url(link_str)diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.pyindex 2a98119fd..df7fa346a 100644--- a/cyberdrop_dl/crawlers/pkmncards.py+++ b/cyberdrop_dl/crawlers/pkmncards.py@@ -7,8 +7,6 @@ from dataclasses import dataclassfrom datetime import datetimefrom typing import TYPE_CHECKING, ClassVar-from bs4 import BeautifulSoup-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError@@ -17,7 +15,7 @@ from cyberdrop_dl.utils.dates import TimeStamp, to_timestampfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup, Tag+ from bs4 import Tagfrom cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -146,13 +144,12 @@ class PkmncardsCrawler(Crawler):download_url = self.parse_url(link_str)simple_card = create_simple_card(title, download_url)new_scrape_item = scrape_item.create_child(card_page_url)- self.manager.task_group.create_task(self.handle_simple_card(new_scrape_item, simple_card))+ self.create_task(self.handle_simple_card(new_scrape_item, simple_card))scrape_item.add_children()@error_handling_wrapperasync def card(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)name = css.select_one_get_text(soup, _SELECTORS.CARD_NAME)number = css.select_one_get_text(soup, _SELECTORS.CARD_NUMBER)@@ -177,8 +174,7 @@ class PkmncardsCrawler(Crawler):async def handle_simple_card(self, scrape_item: ScrapeItem, simple_card: SimpleCard) -> None:@error_handling_wrapperasync def get_card_set(self, scrape_item: ScrapeItem) -> CardSet:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)return create_set(soup)async with self.set_locks[simple_card.set_abbr]:diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.pyindex 010a55e75..fba91091d 100644--- a/cyberdrop_dl/crawlers/pmvhaven.py+++ b/cyberdrop_dl/crawlers/pmvhaven.py@@ -1,7 +1,8 @@from __future__ import annotations+import itertoolsimport json-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -74,17 +75,20 @@ class PMVHavenCrawler(Crawler):# Playlist# TODO: add pagination support for user playlists- add_headers = {"Content-Type": "text/plain;charset=UTF-8"}- add_data = json.dumps({"profile": username, "mode": "GetUser"})- async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.DOMAIN, api_url, data=add_data, headers=add_headers)++ json_resp: dict[str, Any] = await self.request_json(+ api_url,+ method="POST",+ data=json.dumps({"profile": username, "mode": "GetUser"}),+ headers={"Content-Type": "text/plain;charset=UTF-8"},+ )user_info: dict[str, dict] = json_resp["data"]for playlist in user_info["playlists"]:playlist_id = playlist["_id"]link = PRIMARY_URL / "playlist" / playlist_idnew_scrape_item = scrape_item.create_child(link, new_title_part="Playlists")- self.manager.task_group.create_task(self.playlist(new_scrape_item, add_suffix=False))+ self.create_task(self.playlist(new_scrape_item, add_suffix=False))scrape_item.add_children()@error_handling_wrapper@@ -151,11 +155,9 @@ class PMVHavenCrawler(Crawler):video_id: str = scrape_item.url.name.rsplit("_", 1)[-1]add_data = {"video": video_id, "mode": "InitVideo", "view": True}api_url = API_ENTRYPOINT / "videoInput"- async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.DOMAIN, api_url, data=add_data)-- videos = json_resp.get("video")- video_info: dict = videos[0] if videos else {}+ json_resp: dict[str, Any] = await self.request_json(api_url, method="POST", data=add_data)+ videos: list[dict[str, Any]] = json_resp.get("video", [])+ video_info = videos[0] if videos else {}await self.process_video_info(scrape_item, video_info)async def _generic_search_pager(self, scrape_item: ScrapeItem, add_data: dict, name: str, type: str = "") -> None:@@ -205,18 +207,21 @@ class PMVHavenCrawler(Crawler):is_profile = api_url.name == "profileInput"add_headers = {"Content-Type": "text/plain;charset=UTF-8"} if is_profile else None- while True:+ for page in itertools.count(1):data = {"index": page} | add_dataif is_profile:data = json.dumps(data)- async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.DOMAIN, api_url, data=data, headers=add_headers)+ json_resp: dict[str, Any] = await self.request_json(+ api_url,+ method="POST",+ data=data,+ headers=add_headers,+ )has_videos = bool(json_resp[check_key])if not has_videos:breakyield json_resp- page += 1# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.pyindex 4d8916509..54184a6bc 100644--- a/cyberdrop_dl/crawlers/pornhub.py+++ b/cyberdrop_dl/crawlers/pornhub.py@@ -177,7 +177,7 @@ class PornHubCrawler(Crawler):async def iter_profile_pages(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL, selector: str) -> None:async for soup in self.web_pager(url):for _, new_scrape_item in self.iter_children(scrape_item, soup, selector):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem, album_id: str) -> None:@@ -243,7 +243,7 @@ class PornHubCrawler(Crawler):title = self.create_title(title, playlist_id)scrape_item.setup_as_album(f"{title} [playlist]", album_id=playlist_id)for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.PLAYLIST_VIDEOS, results=results):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:diff --git a/cyberdrop_dl/crawlers/pornpics.py b/cyberdrop_dl/crawlers/pornpics.pyindex 2e334c7e1..50b1062e1 100644--- a/cyberdrop_dl/crawlers/pornpics.py+++ b/cyberdrop_dl/crawlers/pornpics.py@@ -1,6 +1,6 @@from __future__ import annotations-import json+import itertoolsfrom typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths@@ -62,13 +62,13 @@ class PornPicsCrawler(Crawler):title = self.create_title(f"{title} [{collection_type}]")scrape_item.setup_as_profile(title)- async for soup, items in self._web_pager(scrape_item):+ async for soup, items in self._pager(scrape_item):if soup:update_scrape_item(soup)for link in items:new_scrape_item = scrape_item.create_child(link)- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))scrape_item.add_children()@error_handling_wrapper@@ -77,8 +77,7 @@ class PornPicsCrawler(Crawler):gallery_id = gallery_base.rsplit("-", 1)[-1]results = await self.get_album_results(gallery_id)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)scrape_item.url = PRIMARY_URL / "galleries" / gallery_id # canonical URLtitle = css.select_one_get_text(soup, "h1")@@ -96,35 +95,31 @@ class PornPicsCrawler(Crawler):# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~``- async def _web_pager(+ async def _pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[tuple[BeautifulSoup | None, tuple[AbsoluteHttpURL, ...]]]:"""Generator of website pages."""limit = 20- page_url = scrape_item.url.without_query_params("limit", "offset")- offset = int(scrape_item.url.query.get("offset") or 0)+ page_url = scrape_item.url.without_query_params("limit", "offset").with_query(limit=limit)+ init_offset = int(scrape_item.url.query.get("offset") or 0)- async def get_items(current_page: AbsoluteHttpURL) -> tuple[BeautifulSoup | None, tuple[AbsoluteHttpURL, ...]]:- offset = current_page.query.get("offset")+ async def get_items(offset: int) -> tuple[BeautifulSoup | None, tuple[AbsoluteHttpURL, ...]]:+ current_page = page_url.update_query(offset=offset)if not offset: # offset == 0 does not return JSON- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, current_page)+ soup = await self.request_soup(current_page)items = soup.select(IMAGE_SELECTOR)return soup, tuple(self.parse_url(css.get_attr(image, "href")) for image in items)- async with self.request_limiter:- # The response is JSON but the "content-type" is wrong so we have to request it as text- json_resp = await self.client.get_text(self.DOMAIN, current_page)- json_resp = json.loads(json_resp)+ # The response is JSON but the "content-type" is wrong+ async with self.request(current_page) as resp:+ json_resp = await resp.json(content_type=False)return None, tuple(self.parse_url(g["g_url"]) for g in json_resp)- while True:- soup, items = await get_items(page_url)+ for offset in itertools.count(init_offset, limit):+ soup, items = await get_items(offset)yield soup, itemsif len(items) < limit:break- offset += limit- page_url = page_url.update_query(offset=offset, limit=limit)def is_cdn(url: AbsoluteHttpURL) -> bool:diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.pyindex c7c9b2fa7..172a671cb 100644--- a/cyberdrop_dl/crawlers/porntrex.py+++ b/cyberdrop_dl/crawlers/porntrex.py@@ -4,8 +4,6 @@ import itertoolsimport refrom typing import TYPE_CHECKING, ClassVar, NamedTuple-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError@@ -63,9 +61,7 @@ class PorntrexCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLNEXT_PAGE_SELECTOR: ClassVar[str] = _SELECTORS.NEXT_PAGEDOMAIN: ClassVar[str] = "porntrex"-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(3, 10)+ _RATE_LIMIT = 3, 10async def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.name: # The ending slash is necessary or we get a 404 error@@ -84,8 +80,7 @@ class PorntrexCrawler(Crawler):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:album_id = scrape_item.url.parts[2]- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)if "This album is a private album" in soup.text:raise ScrapeError(401, "Private album")@@ -108,8 +103,7 @@ class PorntrexCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)video = get_video_info(soup)link = self.parse_url(video.url)@@ -122,8 +116,7 @@ class PorntrexCrawler(Crawler):@error_handling_wrapperasync def collection(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)if "models" in scrape_item.url.parts:title: str = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME).title()@@ -151,7 +144,7 @@ class PorntrexCrawler(Crawler):last_page: int = int(last_page_tag[-1].get_text(strip=True)) if last_page_tag else 1for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEOS_OR_ALBUMS):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))await self.proccess_additional_pages(scrape_item, last_page)@@ -161,11 +154,10 @@ class PorntrexCrawler(Crawler):elif "members" in scrape_item.url.parts:albums_url = scrape_item.url / "albums"- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, albums_url)+ soup = await self.request_soup(albums_url)for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUMS, new_title_part="albums"):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))async def proccess_additional_pages(self, scrape_item: ScrapeItem, last_page: int, **kwargs: str) -> None:block_id: str = "list_videos_common_videos_list_norm"@@ -200,11 +192,10 @@ class PorntrexCrawler(Crawler):if page > last_page:breakpage_url = page_url.update_query({from_param_name: page})- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, page_url)+ soup = await self.request_soup(page_url)for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEOS_OR_ALBUMS):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))def get_video_info(soup: BeautifulSoup) -> Video:diff --git a/cyberdrop_dl/crawlers/postimg.py b/cyberdrop_dl/crawlers/postimg.pyindex 6a2a61b64..bf1280470 100644--- a/cyberdrop_dl/crawlers/postimg.py+++ b/cyberdrop_dl/crawlers/postimg.py@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemPRIMARY_URL = AbsoluteHttpURL("https://postimages.org/")@@ -39,13 +37,15 @@ class PostImgCrawler(Crawler):async def album(self, scrape_item: ScrapeItem) -> None:data = {"action": "list", "album": scrape_item.url.raw_name, "page": 0}title: str = ""+ album_id = scrape_item.url.parts[2]for page in itertools.count(1):- data["page"] = page- async with self.request_limiter:- json_resp = await self.client.post_data(self.DOMAIN, API_ENTRYPOINT, data=data)+ json_resp = await self.request_json(+ API_ENTRYPOINT,+ method="POST",+ data=data | {"page": page},+ )if not title:- album_id = scrape_item.url.parts[2]title = self.create_title(scrape_item.url.name, album_id)scrape_item.setup_as_album(title, album_id=album_id)@@ -64,8 +64,7 @@ class PostImgCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_str: str = css.select_one_get_attr(soup, DOWNLOAD_BUTTON_SELECTOR, "href")link = self.parse_url(link_str).with_query(None)diff --git a/cyberdrop_dl/crawlers/realbooru.py b/cyberdrop_dl/crawlers/realbooru.pyindex 7791481d5..a9d94458a 100644--- a/cyberdrop_dl/crawlers/realbooru.py+++ b/cyberdrop_dl/crawlers/realbooru.py@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -51,12 +49,11 @@ class RealBooruCrawler(Crawler):async for soup in self.web_pager(scrape_item.url, relative_to=scrape_item.url):for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.CONTENT):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_str = css.select_one_get_attr(soup, _SELECTORS.IMAGE_OR_VIDEO, "src")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)diff --git a/cyberdrop_dl/crawlers/realdebrid.py b/cyberdrop_dl/crawlers/realdebrid.pyindex b514e1f26..305e6e42a 100644--- a/cyberdrop_dl/crawlers/realdebrid.py+++ b/cyberdrop_dl/crawlers/realdebrid.py@@ -14,9 +14,6 @@ import refrom re import Patternfrom typing import TYPE_CHECKING, Any, ClassVar-import aiohttp-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import RealDebridError@@ -24,6 +21,7 @@ from cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:+ from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.data_structures.url_objects import ScrapeItemPRIMARY_URL = AbsoluteHttpURL("https://real-debrid.com")@@ -74,12 +72,12 @@ class RealDebridCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "real-debrid"FOLDER_DOMAIN: ClassVar[str] = "RealDebrid"+ _RATE_LIMIT = 250, 60def __post_init__(self) -> None:self._api_token = token = self.manager.auth_config.realdebrid.api_keyself._supported_folder_url_regex: Patternself._supported_url_regex: Pattern- self.request_limiter = AsyncLimiter(250, 60)self.disabled = not bool(token)self._headers = {"Authorization": f"Bearer {token}", "User-Agent": "CyberDrop-DL"}@@ -103,11 +101,13 @@ class RealDebridCrawler(Crawler):if self.disabled:returntry:- files_r, folders_r = await asyncio.gather(- self._api_request("hosts/regex"), self._api_request("hosts/regexFolder")+ responses: tuple[list[str], list[str]] = await asyncio.gather(+ self._api_request("hosts/regex"),+ self._api_request("hosts/regexFolder"),)- file_regex: list[str] = [pattern[1:-1] for pattern in files_r]- folder_regex: list[str] = [pattern[1:-1] for pattern in folders_r]++ file_regex = [pattern[1:-1] for pattern in responses[0]]+ folder_regex = [pattern[1:-1] for pattern in responses[1]]self._supported_url_regex = re.compile("|".join(file_regex + folder_regex))self._supported_folder_url_regex = re.compile("|".join(folder_regex))except Exception as e:@@ -159,12 +159,27 @@ class RealDebridCrawler(Crawler):async def _api_request(self, path: str, /, **data: Any) -> Any:method = "POST" if data else "GET"- data_ = data or None- async with self.request_limiter, self.client._session.disabled():- response = await self.client._session.request(- method, _API_ENTRYPOINT / path, headers=self._headers, data=data_- )- return await _handle_api_response(response)++ async with self.request(+ _API_ENTRYPOINT / path,+ method=method,+ headers=self._headers,+ data=data or None,+ cache_disabled=True,+ ) as resp:+ return await self._handle_api_response(resp)++ async def _handle_api_response(self, response: AbstractResponse) -> Any:+ if "json" in response.content_type:+ json_resp: dict[str, Any] = await response.json()+ if code := json_resp.get("error_code"):+ code = 7 if code == 16 else code+ msg = _ERROR_CODES.get(code, "Unknown error")+ raise RealDebridError(response.url, code, msg) from None+ else:+ return json_resp++ await self.client.client_manager.check_http_status(response)def _guess_folder(url: AbsoluteHttpURL) -> str:@@ -235,22 +250,3 @@ def _flatten_url(original_url: AbsoluteHttpURL, host: str) -> AbsoluteHttpURL:flatten_url = flatten_url / "frag" / original_url.fragmentreturn flatten_url---async def _handle_api_response(response: aiohttp.ClientResponse) -> Any:- try:- json_resp: dict[str, Any] = await response.json()- try:- response.raise_for_status()- except aiohttp.ClientResponseError:- code = json_resp.get("error_code")- if not code:- raise- code = 7 if code == 16 else code- msg = _ERROR_CODES.get(code, "Unknown error")- raise RealDebridError(response.url, code, msg) from None- else:- return json_resp- except AttributeError:- response.raise_for_status()- return await response.text()diff --git a/cyberdrop_dl/crawlers/reddit.py b/cyberdrop_dl/crawlers/reddit.pyindex 6e0ea74c5..d78824ab5 100644--- a/cyberdrop_dl/crawlers/reddit.py+++ b/cyberdrop_dl/crawlers/reddit.py@@ -4,7 +4,6 @@ from contextlib import contextmanagerfrom typing import TYPE_CHECKING, ClassVar, NotRequired, TypedDictimport asyncprawcore-from aiolimiter import AsyncLimiterfrom asyncpraw import Redditfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths@@ -52,11 +51,11 @@ class RedditCrawler(Crawler):DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{title}"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "reddit"+ _RATE_LIMIT = 5, 1def __post_init__(self) -> None:self.reddit_personal_use_script = self.manager.config_manager.authentication_data.reddit.personal_use_scriptself.reddit_secret = self.manager.config_manager.authentication_data.reddit.secret- self.request_limiter = AsyncLimiter(5, 1)self.logged_in = Falseasync def async_startup(self) -> None:@@ -167,7 +166,10 @@ class RedditCrawler(Crawler):try:filename, ext = self.get_filename_and_ext(url.name)except NoExtensionError:- scrape_item.url = url = await self.get_final_location(url)+ async with self.request(url) as resp:+ if url == resp.url:+ raise+ scrape_item.url = url = resp.urlwith asyncpraw_error_handle():post = await self._reddit.submission(url=str(url))@@ -175,17 +177,6 @@ class RedditCrawler(Crawler):await self.handle_file(url, scrape_item, filename, ext)- async def get_final_location(self, url) -> AbsoluteHttpURL:- headers = await self.client.get_head(self.DOMAIN, url)- content_type = headers.get("Content-Type", "")- if any(s in content_type.lower() for s in ("html", "text")):- response, _ = await self.client._get_response_and_soup(self.DOMAIN, url)- return AbsoluteHttpURL(response.url)- location = headers.get("location")- if not location:- raise ScrapeError(422)- return self.parse_url(location)-@error_handling_wrapperasync def check_login(self, _) -> None:if not (self.reddit_personal_use_script and self.reddit_secret):diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.pyindex 06ef47d28..f660b18d6 100644--- a/cyberdrop_dl/crawlers/redgifs.py+++ b/cyberdrop_dl/crawlers/redgifs.py@@ -16,7 +16,7 @@ if TYPE_CHECKING:# Primary URL needs `www.` to prevent redirectPRIMARY_URL = AbsoluteHttpURL("https://www.redgifs.com/")-API_ENTRYPOINT = AbsoluteHttpURL("https://api.redgifs.com/")+API_ENTRYPOINT = AbsoluteHttpURL("https://api.redgifs.com/v2")class Links(TypedDict, total=False):@@ -53,7 +53,7 @@ class RedGifsCrawler(Crawler):self.headers = {}async def async_startup(self) -> None:- await self.get_auth_token(API_ENTRYPOINT / "v2/auth/temporary")+ await self.get_auth_token(API_ENTRYPOINT / "auth/temporary")async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:@@ -64,7 +64,7 @@ class RedGifsCrawler(Crawler):if self.is_self_subdomain(scrape_item.url) and len(scrape_item.url.parts) == 2:scrape_item.url = _canonical_url(scrape_item.url.name)- self.manager.task_group.create_task(self.run(scrape_item))+ self.create_task(self.run(scrape_item))returnraise ValueError@@ -83,11 +83,8 @@ class RedGifsCrawler(Crawler):async def _user_profile_pager(self, user_id: str) -> AsyncIterable[list[Gif]]:total_pages: int | None = Nonefor page in itertools.count(1):- async with self.request_limiter:- api_url = API_ENTRYPOINT / "v2/users" / user_id / "search"- api_url = api_url.with_query(order="new", page=page)- json_resp: dict[str, Any] = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)-+ api_url = (API_ENTRYPOINT / "users" / user_id / "search").with_query(order="new", page=page)+ json_resp: dict[str, Any] = await self.request_json(api_url, headers=self.headers)gifs_in_current_page = [Gif.from_dict(gif) for gif in json_resp["gifs"]]yield gifs_in_current_page@@ -107,10 +104,8 @@ class RedGifsCrawler(Crawler):returnscrape_item.url = canonical_url- async with self.request_limiter:- api_url = API_ENTRYPOINT / "v2/gifs" / post_id- json_resp: dict[str, dict] = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)-+ api_url = API_ENTRYPOINT / "gifs" / post_id+ json_resp: dict[str, dict] = await self.request_json(api_url, headers=self.headers)gif = Gif.from_dict(json_resp["gif"])if gif.title:scrape_item.setup_as_album(self.create_title(gif.title))@@ -123,8 +118,7 @@ class RedGifsCrawler(Crawler):@error_handling_wrapperasync def get_auth_token(self, token_url: AbsoluteHttpURL) -> None:- async with self.request_limiter:- json_obj = await self.client.get_json(self.DOMAIN, token_url)+ json_obj: dict[str, Any] = await self.request_json(token_url)token: str = json_obj["token"]self.headers = {"Authorization": f"Bearer {token}"}diff --git a/cyberdrop_dl/crawlers/rule34vault.py b/cyberdrop_dl/crawlers/rule34vault.pyindex d442dd8e6..fb9f69015 100644--- a/cyberdrop_dl/crawlers/rule34vault.py+++ b/cyberdrop_dl/crawlers/rule34vault.py@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -52,8 +50,7 @@ class Rule34VaultCrawler(Crawler):for page in itertools.count(init_page):url = scrape_item.url.with_query(page=page)n_images = 0- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, url)+ soup = await self.request_soup(url)if not title:if is_playlist:@@ -67,7 +64,7 @@ class Rule34VaultCrawler(Crawler):for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.CONTENT):n_images += 1- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))if n_images < 30:break@@ -78,8 +75,7 @@ class Rule34VaultCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)if date_tag := soup.select_one(_SELECTORS.DATE):scrape_item.possible_datetime = self.parse_date(date_tag.text, "%b %d, %Y, %I:%M:%S %p")diff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.pyindex c37ea77f4..13a331373 100644--- a/cyberdrop_dl/crawlers/rule34video.py+++ b/cyberdrop_dl/crawlers/rule34video.py@@ -84,7 +84,7 @@ class Rule34VideoCrawler(Crawler):scrape_item.setup_as_album(title)for _, new_scrape_item in self.iter_children(scrape_item, soup, PLAYLIST_ITEM_SELECTOR):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:@@ -94,8 +94,7 @@ class Rule34VideoCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)scrape_item.url = canonical_urlvideo = _parse_video(soup)diff --git a/cyberdrop_dl/crawlers/rule34xxx.py b/cyberdrop_dl/crawlers/rule34xxx.pyindex c538c2b4a..800c34966 100644--- a/cyberdrop_dl/crawlers/rule34xxx.py+++ b/cyberdrop_dl/crawlers/rule34xxx.py@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemPRIMARY_URL = AbsoluteHttpURL("https://rule34.xxx")@@ -53,12 +51,11 @@ class Rule34XXXCrawler(Crawler):scrape_item.setup_as_album(title)for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.CONTENT):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)date_str = css.select_one_get_text(soup, _SELECTORS.DATE).removeprefix("Posted: ")scrape_item.possible_datetime = self.parse_date(date_str)diff --git a/cyberdrop_dl/crawlers/saint.py b/cyberdrop_dl/crawlers/saint.pyindex 938f039c4..408e11bd7 100644--- a/cyberdrop_dl/crawlers/saint.py+++ b/cyberdrop_dl/crawlers/saint.py@@ -51,8 +51,7 @@ class SaintCrawler(Crawler):async def album(self, scrape_item: ScrapeItem) -> None:album_id = scrape_item.url.parts[2]results = await self.get_album_results(album_id)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)title_portion = css.select_one_get_text(soup, "title").rsplit(" - Saint Video Hosting")[0].strip()if not title_portion:@@ -66,7 +65,7 @@ class SaintCrawler(Crawler):link = self.parse_url(link_str)if not self.check_album_results(link, results):new_scrape_item = scrape_item.create_child(link)- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))scrape_item.add_children()@error_handling_wrapper@@ -74,8 +73,7 @@ class SaintCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)try:link_str: str = css.select_one_get_attr(soup, _SELECTORS.EMBED_SRC, "src")except AssertionError:@@ -88,8 +86,7 @@ class SaintCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)try:link_str: str = css.select_one_get_attr(soup, _SELECTORS.DOWNLOAD_BUTTON, "href")except (AttributeError, css.SelectorError):diff --git a/cyberdrop_dl/crawlers/scrolller.py b/cyberdrop_dl/crawlers/scrolller.pyindex 484f8cc60..8b35239cf 100644--- a/cyberdrop_dl/crawlers/scrolller.py+++ b/cyberdrop_dl/crawlers/scrolller.py@@ -72,10 +72,14 @@ class ScrolllerCrawler(Crawler):while True:request_body["variables"]["iterator"] = iterator- data: dict[str, dict] = await self.client.post_data(- self.DOMAIN, API_ENTRYPOINT, data=json.dumps(request_body)- )- items: list[dict] = data["data"]["getSubreddit"]["children"]["items"] if data else []+ data: dict[str, dict] = (+ await self.request_json(+ API_ENTRYPOINT,+ method="POST",+ data=json.dumps(request_body),+ )+ )["data"]+ items: list[dict] = data["getSubreddit"]["children"]["items"] if data else []if not items:break@@ -94,7 +98,7 @@ class ScrolllerCrawler(Crawler):scrape_item.add_children()prev_iterator = iterator- iterator = data["data"]["getSubreddit"]["children"]["iterator"]+ iterator = data["getSubreddit"]["children"]["iterator"]if iterator is None or iterator == prev_iterator or iterations > 0:breakdiff --git a/cyberdrop_dl/crawlers/send_now.py b/cyberdrop_dl/crawlers/send_now.pyindex dd5542ce4..a8ac9b288 100644--- a/cyberdrop_dl/crawlers/send_now.py+++ b/cyberdrop_dl/crawlers/send_now.py@@ -7,8 +7,6 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from curl_cffi.requests.models import Response as CurlResponse-from cyberdrop_dl.data_structures.url_objects import ScrapeItemPRIMARY_URL = AbsoluteHttpURL("https://send.now/")@@ -29,22 +27,35 @@ class SendNowCrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:file_id = scrape_item.url.name- data = {"op": "download2", "id": file_id, "rand": "", "referer": "", "method_free": "", "method_premium": ""}- params = {"allow_redirects": False, "stream": True}-- async with self.request_limiter:- response: CurlResponse = await self.client.post_data_cffi(- self.DOMAIN, scrape_item.url, data=data, request_params=params- )- debrid_link = self.parse_url(response.headers.get("location"))+ async with self.request(+ scrape_item.url,+ method="POST",+ impersonate=True,+ data={+ "op": "download2",+ "id": file_id,+ "rand": "",+ "referer": "",+ "method_free": "",+ "method_premium": "",+ },+ request_params={"allow_redirects": False, "stream": True},+ ) as resp:+ debrid_link = resp.location++ assert debrid_linkfilename, ext = self.get_filename_and_ext(debrid_link.name, assume_ext=".zip")await self.handle_file(scrape_item.url, scrape_item, filename, ext, debrid_link=debrid_link)- async def get_cookies(self, scrape_item: ScrapeItem) -> None:+ async def _get_cookies(self, scrape_item: ScrapeItem) -> None:+ if self.got_cookies:+ returnasync with self.startup_lock:- if not self.got_cookies:- async with self.request_limiter:- await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)- cookies = self.manager.client_manager.cookies.filter_cookies(PRIMARY_URL)- self.got_cookies = bool(cookies)+ if self.got_cookies:+ return++ async with self.request(scrape_item.url, impersonate=True):+ pass+ cookies = self.manager.client_manager.cookies.filter_cookies(PRIMARY_URL)+ self.got_cookies = bool(cookies)diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.pyindex 1f1acce9d..a75857bb7 100644--- a/cyberdrop_dl/crawlers/sendvid.py+++ b/cyberdrop_dl/crawlers/sendvid.py@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_partsif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemPRIMARY_URL = AbsoluteHttpURL("https://sendvid.com/")@@ -38,8 +36,7 @@ class SendVidCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)title = css.select_one_get_text(soup, TITLE_SELECTOR)try:diff --git a/cyberdrop_dl/crawlers/sex_dot_com.py b/cyberdrop_dl/crawlers/sex_dot_com.pyindex 27fe6e556..cf8c9a643 100644--- a/cyberdrop_dl/crawlers/sex_dot_com.py+++ b/cyberdrop_dl/crawlers/sex_dot_com.py@@ -1,5 +1,6 @@from __future__ import annotations+import itertoolsfrom typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths@@ -14,6 +15,7 @@ if TYPE_CHECKING:PRIMARY_URL = AbsoluteHttpURL("https://sex.com")API_URL = AbsoluteHttpURL("https://iframe.sex.com/api/")+_SHORTS_URL = AbsoluteHttpURL("https://sex.com/en/shorts")class SexDotComCrawler(Crawler):@@ -30,18 +32,14 @@ class SexDotComCrawler(Crawler):return await self.profile(scrape_item)raise ValueError- async def shorts_profile_paginator(self, scrape_item: ScrapeItem) -> AsyncGenerator[dict]:+ async def shorts_profile_paginator(self, scrape_item: ScrapeItem) -> AsyncGenerator[dict[str, Any]]:username = scrape_item.url.parts[3]- page = 1- while True:- posts_api_url = API_URL / "feed" / "listUserItems"- query = {"pageSize": 300, "pageNumber": page, "visibility": "public", "username": username}- posts_api_url = posts_api_url.with_query(query)- async with self.request_limiter:- json_data = await self.client.get_json(self.DOMAIN, posts_api_url)+ api_url = (API_URL / "feed" / "listUserItems").with_query(pageSize=300, visibility="public", username=username)+ for page in itertools.count(1):+ json_data: dict[str, Any] = await self.request_json(api_url.with_query(pageNumber=page))if scrape_item.album_id is None:- user_id = json_data["page"]["items"][0]["media"]["user"]["userUid"]+ user_id: str = json_data["page"]["items"][0]["media"]["user"]["userUid"]title = self.create_title(username, user_id)scrape_item.setup_as_profile(title, album_id=user_id)@@ -49,32 +47,28 @@ class SexDotComCrawler(Crawler):if not json_data["page"]["pageInfo"]["hasNextPage"]:break- page += 1async def get_media(self, scrape_item: ScrapeItem) -> dict[str, Any]:"""Gets media from its relative URL."""relative_url = "/".join(scrape_item.url.parts[3:])- data_url = API_URL / "media" / "getMedia"- data_url = data_url.with_query(relativeUrl=relative_url)- async with self.request_limiter:- json_data = await self.client.get_json(self.DOMAIN, data_url)- return json_data["media"]+ data_url = (API_URL / "media" / "getMedia").with_query(relativeUrl=relative_url)+ return (await self.request_json(data_url))["media"]@error_handling_wrapperasync def handle_media(self, scrape_item: ScrapeItem, item: dict[str, Any] | None) -> None:real_item = item or await self.get_media(scrape_item)relative_url = real_item["relativeUrl"]- canonical_url = AbsoluteHttpURL("https://sex.com/en/shorts") / relative_url+ canonical_url = _SHORTS_URL / relative_urlif await self.check_complete_from_referer(canonical_url):returnfileType: str = real_item.get("fileType") or real_item["mediaType"]if fileType.startswith("image"):- media_url = AbsoluteHttpURL(real_item["fullPath"]).with_query(optimizer="image", width=1200)+ media_url = self.parse_url(real_item["fullPath"]).with_query(optimizer="image", width=1200)filename, ext = f"{real_item['pictureUid']}.jpg", "jpg"elif fileType.startswith("video"):- media_url = AbsoluteHttpURL(real_item["sources"][0]["fullPath"])+ media_url = self.parse_url(real_item["sources"][0]["fullPath"])filename, ext = self.get_filename_and_ext(media_url.name)else:returndiff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.pyindex 085d5e175..9b47a572d 100644--- a/cyberdrop_dl/crawlers/spankbang.py+++ b/cyberdrop_dl/crawlers/spankbang.py@@ -89,8 +89,7 @@ class SpankBangCrawler(Crawler):title: str = ""for page in itertools.count(1):- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, page_url)+ soup = await self.request_soup(page_url, impersonate=True)# Get full playlist info + title from the soupplaylist = PlaylistInfo.from_url(page_url, soup)@@ -102,7 +101,7 @@ class SpankBangCrawler(Crawler):for _, new_scrape_item in self.iter_children(scrape_item, soup, VIDEOS_SELECTOR, results=results):n_videos += 1- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))if n_videos < 100:break@@ -117,9 +116,7 @@ class SpankBangCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)-+ soup = await self.request_soup(scrape_item.url, impersonate=True)was_removed = soup.select_one(VIDEO_REMOVED_SELECTOR)if was_removed or "This video is no longer available" in soup.text:raise ScrapeError(410)diff --git a/cyberdrop_dl/crawlers/streamable.py b/cyberdrop_dl/crawlers/streamable.pyindex aaa944b2e..c2575ad23 100644--- a/cyberdrop_dl/crawlers/streamable.py+++ b/cyberdrop_dl/crawlers/streamable.py@@ -42,8 +42,7 @@ class StreamableCrawler(Crawler):returnajax_url = AJAX_ENTRYPOINT / video_id- async with self.request_limiter:- json_resp: dict[str, Any] = await self.client.get_json(self.DOMAIN, ajax_url)+ json_resp: dict[str, Any] = await self.request_json(ajax_url)status: int = json_resp["status"]if status != STATUS_OK:diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.pyindex 493261e89..232c435fa 100644--- a/cyberdrop_dl/crawlers/tokyomotion.py+++ b/cyberdrop_dl/crawlers/tokyomotion.py@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_partsif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItemPRIMARY_URL = AbsoluteHttpURL("https://www.tokyomotion.net")@@ -82,8 +80,7 @@ class TokioMotionCrawler(Crawler):returnvideo_id = scrape_item.url.parts[2]- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)src = soup.select_one(_SELECTORS.VIDEO_SRC)if not src:@@ -104,8 +101,7 @@ class TokioMotionCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)img = soup.select_one(_SELECTORS.IMAGE)if not img:@@ -205,8 +201,7 @@ class TokioMotionCrawler(Crawler):return "favorite"if "album" in scrape_item.url.parts and len(scrape_item.url.parts) > 3:return scrape_item.url.parts[3]- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)return css.select_one_get_text(soup, _SELECTORS.ALBUM_TITLE)def add_user_title(self, scrape_item: ScrapeItem) -> None:diff --git a/cyberdrop_dl/crawlers/toonily.py b/cyberdrop_dl/crawlers/toonily.pyindex b3c89864c..115f4c36e 100644--- a/cyberdrop_dl/crawlers/toonily.py+++ b/cyberdrop_dl/crawlers/toonily.py@@ -37,8 +37,7 @@ class ToonilyCrawler(Crawler):@error_handling_wrapperasync def series(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)title_tag = css.select_one(soup, Selector.SERIES_TITLE)css.decompose(title_tag, "*")@@ -49,8 +48,7 @@ class ToonilyCrawler(Crawler):@error_handling_wrapperasync def chapter(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)series_name, chapter_title = css.select_one_get_text(soup, Selector.CHAPTER_TITLE).split(" - ", 1)if scrape_item.type != FILE_HOST_PROFILE:diff --git a/cyberdrop_dl/crawlers/twpornstars.py b/cyberdrop_dl/crawlers/twpornstars.pyindex cc76ce17e..550103ea6 100644--- a/cyberdrop_dl/crawlers/twpornstars.py+++ b/cyberdrop_dl/crawlers/twpornstars.py@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.crawlers.crawler import SupportedDomainsfrom cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -59,8 +57,7 @@ class TwPornstarsCrawler(TwimgCrawler):async def media(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)if url := soup.select_one(_SELECTORS.PHOTO):url = self.parse_url(css.get_attr(url, "src").replace(":large", ""))await self.photo(scrape_item, url)@@ -85,4 +82,4 @@ class TwPornstarsCrawler(TwimgCrawler):scrape_item.setup_as_album(title)title_created = Truefor _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.THUMBS):- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))diff --git a/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py b/cyberdrop_dl/crawlers/vbulletin/_vbulletin.pyindex 26bc06d9e..055db45d0 100644--- a/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py+++ b/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py@@ -132,9 +132,7 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801scrape_item.add_children()async def get_xml(self, url: AbsoluteHttpURL) -> ElementTree.Element[str]:- async with self.request_limiter:- text = await self.client.get_text(self.DOMAIN, url)- root_xml = ElementTree.XML(text)+ root_xml = ElementTree.XML(await self.request_text(url))if error := root_xml.find("error"):details = error.attrib["details"]error_code = 403 if error.attrib["type"] == "permissions" and "unknown" not in details.casefold() else 422diff --git a/cyberdrop_dl/crawlers/vbulletin/vipergirls.py b/cyberdrop_dl/crawlers/vbulletin/vipergirls.pyindex d5819590b..6a9b8e709 100644--- a/cyberdrop_dl/crawlers/vbulletin/vipergirls.py+++ b/cyberdrop_dl/crawlers/vbulletin/vipergirls.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import ClassVar-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.vbulletin._vbulletin import vBulletinCrawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -16,7 +14,4 @@ class ViperGirlsCrawler(vBulletinCrawler):FOLDER_DOMAIN = "ViperGirls"SUPPORTED_DOMAINS: ClassVar = "viper.click", "vipergirls.to"VBULLETIN_API_ENDPOINT: ClassVar = AbsoluteHttpURL("https://viper.click/vr.php")-- def __post_init__(self) -> None:- super().__post_init__()- self.request_limiter = AsyncLimiter(4, 1)+ _RATE_LIMIT = 4, 1diff --git a/cyberdrop_dl/crawlers/vipr_dot_im.py b/cyberdrop_dl/crawlers/vipr_dot_im.pyindex 6589944fd..4a4c2ab3e 100644--- a/cyberdrop_dl/crawlers/vipr_dot_im.py+++ b/cyberdrop_dl/crawlers/vipr_dot_im.py@@ -34,8 +34,7 @@ class ViprImCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)link_str: str = css.select_one_get_attr(soup, IMG_SELECTOR, "src")link = self.parse_url(link_str)@@ -44,7 +43,7 @@ class ViprImCrawler(Crawler):async def thumbnail(self, scrape_item: ScrapeItem) -> None:scrape_item.url = self.get_canonical_url(scrape_item.url)- self.manager.task_group.create_task(self.run(scrape_item))+ self.create_task(self.run(scrape_item))def get_canonical_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:return PRIMARY_URL / get_image_id(url)diff --git a/cyberdrop_dl/crawlers/wetransfer.py b/cyberdrop_dl/crawlers/wetransfer.pyindex 7a927e3e5..ed4af904e 100644--- a/cyberdrop_dl/crawlers/wetransfer.py+++ b/cyberdrop_dl/crawlers/wetransfer.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport jsonfrom dataclasses import dataclass-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -31,39 +31,26 @@ class WeTransferCrawler(Crawler):async def fetch(self, scrape_item: ScrapeItem) -> None:if "download." in scrape_item.url.host:# We can download but db entry will not have a canonical URL- return await self.direct_link(scrape_item, scrape_item.url)+ return await self.direct_file(scrape_item)await self.file(scrape_item)@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.host == "we.tl":- scrape_item.url = await self.get_final_url(scrape_item)+ scrape_item.url = await self._get_redirect_url(scrape_item.url)file_info = get_file_info(scrape_item.url)if await self.check_complete_from_referer(file_info.download_url):return- async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.DOMAIN, file_info.download_url, json=file_info.json)-+ json_resp: dict[str, Any] = await self.request_json(file_info.download_url, method="POST", json=file_info.json)link_str: str | None = json_resp.get("direct_link")if not link_str:code, msg = parse_error(json_resp)raise ScrapeError(code, message=msg)link = self.parse_url(link_str)- await self.direct_link(scrape_item, link)-- @error_handling_wrapper- async def direct_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:- filename, ext = self.get_filename_and_ext(link.name)- await self.handle_file(link, scrape_item, filename, ext)-- async def get_final_url(self, scrape_item: ScrapeItem) -> AbsoluteHttpURL:- async with self.request_limiter:- headers = await self.client.get_head(self.DOMAIN, scrape_item.url)-- return self.parse_url(headers["location"])+ await self.direct_file(scrape_item, link)@dataclass(frozen=True, order=True)diff --git a/cyberdrop_dl/crawlers/wordpress/_wordpress.py b/cyberdrop_dl/crawlers/wordpress/_wordpress.pyindex b9434c99a..b4a0d92c4 100644--- a/cyberdrop_dl/crawlers/wordpress/_wordpress.py+++ b/cyberdrop_dl/crawlers/wordpress/_wordpress.py@@ -13,7 +13,6 @@ import refrom abc import abstractmethodfrom typing import TYPE_CHECKING, ClassVar, TypeVar, final-from aiolimiter import AsyncLimiterfrom bs4 import BeautifulSoupfrom pydantic import BaseModel@@ -79,15 +78,13 @@ class WordPressBaseCrawler(Crawler, is_abc=True):DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {id} - {title}"WP_USE_REGEX: ClassVar = TrueSUPPORTS_THREAD_RECURSION: ClassVar = False+ _RATE_LIMIT = 3, 1def __init_subclass__(cls, **kwargs) -> None:super().__init_subclass__(**kwargs)assert cls.fetch is WordPressBaseCrawler.fetchassert cls.fetch_with_date_range is WordPressBaseCrawler.fetch_with_date_range- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(3, 1)-@finalasync def fetch(self, scrape_item: ScrapeItem) -> None:date_range = QueryDatetimeRange.from_url(scrape_item.url)@@ -233,8 +230,7 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):await self.filter_post(scrape_item, post)async def __make_request(self, model_cls: type[_ModelT], api_url: AbsoluteHttpURL) -> _ModelT:- async with self.request_limiter:- json_text = await self.client.get_text(self.DOMAIN, api_url)+ json_text = await self.request_text(api_url)return model_cls.model_validate_json(json_text)async def __handle_collection(@@ -264,8 +260,7 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):class WordPressHTMLCrawler(WordPressBaseCrawler, is_generic=True):async def __make_request(self, api_url: AbsoluteHttpURL) -> BeautifulSoup:- async with self.request_limiter:- return await self.client.get_soup(self.DOMAIN, api_url)+ return await self.request_soup(api_url)@error_handling_wrapperasync def category_or_tag(@@ -320,7 +315,7 @@ class WordPressHTMLCrawler(WordPressBaseCrawler, is_generic=True):f"Skipping post {new_scrape_item.url} as it is out of date range. Post date: {date_from_path}")continue- self.manager.task_group.create_task(self.run(new_scrape_item))+ self.create_task(self.run(new_scrape_item))def _match_date_from_path(url_parts: tuple[str, ...]) -> datetime.datetime | None:diff --git a/cyberdrop_dl/crawlers/xbunkr.py b/cyberdrop_dl/crawlers/xbunkr.pyindex af173d4f5..cdbf63d86 100644--- a/cyberdrop_dl/crawlers/xbunkr.py+++ b/cyberdrop_dl/crawlers/xbunkr.py@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -31,8 +29,7 @@ class XBunkrCrawler(Crawler):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)album_id = scrape_item.url.parts[2]title = self.create_title(css.select_one_get_text(soup, TITLE_SELECTOR), album_id)diff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.pyindex df4caf1fe..63c13d236 100644--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py@@ -17,9 +17,7 @@ class F95ZoneCrawler(XenforoCrawler):@error_handling_wrapperasync def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:- async with self.request_limiter:- json_resp = await self.client.post_data(self.DOMAIN, link, data=_confirmation_data)-+ json_resp = await self.request_json(link, method="POST", data=_confirmation_data)if json_resp["status"] == "ok":return self.parse_url(json_resp["msg"])diff --git a/cyberdrop_dl/crawlers/xenforo/simpcity.py b/cyberdrop_dl/crawlers/xenforo/simpcity.pyindex 259b34bc7..0b2d44443 100644--- a/cyberdrop_dl/crawlers/xenforo/simpcity.py+++ b/cyberdrop_dl/crawlers/xenforo/simpcity.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import ClassVar-from aiolimiter import AsyncLimiter-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom .xenforo import XenforoCrawler@@ -17,7 +15,4 @@ class SimpCityCrawler(XenforoCrawler):login_required = FalseIGNORE_EMBEDED_IMAGES_SRC = FalseOLD_DOMAINS: ClassVar[tuple[str, ...]] = ("simpcity.su",)-- def __post_init__(self) -> None:- super().__post_init__()- self.request_limiter = AsyncLimiter(1, 10)+ _RATE_LIMIT = 1, 20diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.pyindex 2dfebdf95..dae4dab15 100644--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py@@ -124,7 +124,13 @@ class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):attempt += 1await asyncio.sleep(wait_time)data = parse_login_form(text) | credentials- _ = await self.client._post_data(self.DOMAIN, login_url / "login", data=data, cache_disabled=True)+ async with self.request(+ login_url / "login",+ method="POST",+ data=data,+ cache_disabled=True,+ ):+ passawait asyncio.sleep(wait_time)text, logged_in = await self.check_login_with_request(login_url)if logged_in:@@ -132,6 +138,7 @@ class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):returnexcept TimeoutError:continue+msg = f"Failed to login on {self.FOLDER_DOMAIN} after {retries} attempts"raise LoginError(message=msg)diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex 4b46f415e..81f24b441 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -6,8 +6,6 @@ import itertoolsimport jsonfrom typing import TYPE_CHECKING, Any, ClassVar, NamedTuple-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_url, xor_decrypt@@ -53,9 +51,9 @@ class XhamsterCrawler(Crawler):NEXT_PAGE_SELECTOR = Selector.NEXT_PAGEDOMAIN = "xhamster"FOLDER_DOMAIN = "xHamster"+ _RATE_LIMIT = 4, 1def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(4, 1)self._seen_hosts: set[str] = set()def _disable_ai_title_translations(self, url: AbsoluteHttpURL) -> None:@@ -208,9 +206,7 @@ class XhamsterCrawler(Crawler):async def _get_window_initials(self, url: AbsoluteHttpURL) -> dict[str, Any]:self._disable_ai_title_translations(url)- async with self.request_limiter:- content = await self.client.get_text(self.DOMAIN, url)-+ content = await self.request_text(url)initials = get_text_between(content, "window.initials=", ";</script>")return json.loads(initials)diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.pyindex 4ff07440d..a2a4a566a 100644--- a/cyberdrop_dl/crawlers/xvideos.py+++ b/cyberdrop_dl/crawlers/xvideos.py@@ -179,28 +179,35 @@ class XVideosCrawler(Crawler):self.create_task(self.direct_file(scrape_item, src))async def _get_soup(self, url: AbsoluteHttpURL) -> BeautifulSoup:- async with self._domain_locks[url.host]:- if url.host not in self._seen_domains:- await self._disable_auto_translated_titles(url.origin())- self._seen_domains.add(url.host)+ if url.host not in self._seen_domains:+ async with self._domain_locks[url.host]:+ if url.host not in self._seen_domains:+ await self._disable_auto_translated_titles(url.origin())+ self._seen_domains.add(url.host)- async with self.request_limiter:- return await self.client.get_soup(self.DOMAIN, url, self._headers)+ return await self.request_soup(url, headers=self._headers)async def _disable_auto_translated_titles(self, origin: AbsoluteHttpURL) -> None:- async with self.request_limiter:- await self.client._get(self.DOMAIN, origin / "change-language/en", self._headers)- json_resp = await self.client.post_data(- self.DOMAIN, origin / "account/feature-disabled", self._headers, data={"featureid": "at"}- )+ async with self.request(origin / "change-language/en", headers=self._headers):+ pass++ json_resp: dict[str, Any] = await self.request_json(+ origin / "account/feature-disabled",+ method="POST",+ headers=self._headers,+ data={"featureid": "at"},+ )if json_resp["code"] != 0:self.disabled = Trueraise ScrapeError(json_resp["code"])async def _iter_api_pages(self, scrape_item: ScrapeItem, api_url: AbsoluteHttpURL, new_part: str) -> None:for page in itertools.count(0):- async with self.request_limiter:- json_resp: dict[str, Any] = await self.client.post_data(self.DOMAIN, api_url / str(page), self._headers)+ json_resp: dict[str, Any] = await self.request_json(+ api_url / str(page),+ method="POST",+ headers=self._headers,+ )if json_resp["code"] != 0:raise ScrapeError(json_resp["code"])diff --git a/cyberdrop_dl/crawlers/xxxbunker.py b/cyberdrop_dl/crawlers/xxxbunker.pyindex 31ccdbe8c..1db7c5cf1 100644--- a/cyberdrop_dl/crawlers/xxxbunker.py+++ b/cyberdrop_dl/crawlers/xxxbunker.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar-from aiolimiter import AsyncLimiter-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeError@@ -34,9 +32,7 @@ class XXXBunkerCrawler(Crawler):FOLDER_DOMAIN: ClassVar[str] = "XXXBunker"NEXT_PAGE_SELECTOR = Selector.NEXT_PAGE_DOWNLOAD_SLOTS: ClassVar[int | None] = 2-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(1, 6)+ _RATE_LIMIT = 1, 6async def async_startup(self) -> None:self.update_cookies({"ageconfirm": "True"})@@ -57,16 +53,11 @@ class XXXBunkerCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return- async with self.request_limiter:- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)-+ soup = await self.request_soup(scrape_item.url)_check_video_is_available(soup)title = open_graph.title(soup)iframe_url = self.parse_url(css.select_one_get_attr(soup, Selector.VIDEO_IFRAME, "data-src"))-- async with self.request_limiter:- iframe_soup = await self.client.get_soup(self.DOMAIN, iframe_url)-+ iframe_soup = await self.request_soup(iframe_url)src = self.parse_url(css.select_one_get_attr(iframe_soup, "source", "src"))video_id = iframe_url.namecustom_filename = self.create_custom_filename(title, ".mp4", file_id=video_id)diff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.pyindex 6b9b53412..f08881e08 100644--- a/cyberdrop_dl/crawlers/yandex_disk.py+++ b/cyberdrop_dl/crawlers/yandex_disk.py@@ -71,9 +71,7 @@ class YandexDiskCrawler(Crawler):returnasync with self.request_context():- soup: BeautifulSoup = await self.client.get_soup(- self.DOMAIN, scrape_item.url, _DEFAULT_HEADERS, cache_disabled=True- )+ soup = await self.request_soup(scrape_item.url, headers=_DEFAULT_HEADERS, cache_disabled=True)item_info = get_item_info(soup)assert is_single_item(item_info)@@ -90,9 +88,7 @@ class YandexDiskCrawler(Crawler):scrape_item.url = canonical_urlasync with self.request_context():- soup: BeautifulSoup = await self.client.get_soup(- self.DOMAIN, scrape_item.url, _DEFAULT_HEADERS, cache_disabled=True- )+ soup = await self.request_soup(scrape_item.url, headers=_DEFAULT_HEADERS, cache_disabled=True)item_info = get_item_info(soup)del soup@@ -123,8 +119,7 @@ class YandexDiskCrawler(Crawler):@contextlib.asynccontextmanagerasync def request_context(self) -> AsyncGenerator[None]:try:- async with self.request_limiter:- yield+ yieldexcept DownloadError as e:if e.status in (400, 403):raise DDOSGuardError from None@@ -146,7 +141,12 @@ class YandexDiskCrawler(Crawler):api_url = DOWNLOAD_API_ENTRYPOINT.with_host(scrape_item.url.host)async with self.request_context():- json_resp: dict[str, Any] = await self.client.post_data(self.DOMAIN, api_url, headers, data=file.post_data)+ json_resp: dict[str, Any] = await self.request_json(+ api_url,+ method="POST",+ headers=headers,+ data=file.post_data,+ )new_sk = json_resp.get("new_sk")if new_sk:@@ -155,9 +155,7 @@ class YandexDiskCrawler(Crawler):error = json_resp.get("error")if error:- # The error format is dynamic but they are short- # We can log them to the main file- raise ScrapeError(422, message=json.dumps(json_resp))+ raise ScrapeError(422, message=json.dumps(json_resp)[:50])self.log_debug(json_resp)scrape_item.possible_datetime = file.modifieddiff --git a/cyberdrop_dl/crawlers/youjizz.py b/cyberdrop_dl/crawlers/youjizz.pyindex ab3a08cd5..5b50e346f 100644--- a/cyberdrop_dl/crawlers/youjizz.py+++ b/cyberdrop_dl/crawlers/youjizz.py@@ -60,8 +60,7 @@ class YouJizzCrawler(Crawler):if await self.check_complete_from_referer(canonical_url):return- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.request_soup(scrape_item.url)scrape_item.url = canonical_urlvideo = _parse_video(soup)diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex e1415a6e6..505bc3a38 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -477,7 +477,7 @@ class MegaApi:self.root_id: str = ""self.inbox_id: str = ""self.trashbin_id: str = ""- self.request_limiter = AsyncLimiter(100, 60)+ self._limiter = AsyncLimiter(100, 60)self._files = {}self.shared_keys: SharedkeysDict@@ -498,7 +498,7 @@ class MegaApi:else:data: list[AnyDict] = data_input- async with self.request_limiter, self.session.disabled():+ async with self._limiter, self.session.disabled():response = await self.session.post(self.entrypoint, params=params, json=data, timeout=self.timeout, headers=self.default_headers)@@ -513,7 +513,7 @@ class MegaApi:log("[MegaNZ] Solving xhashcash login challenge, this could take a few seconds...")xhashcash_token = await generate_hashcash_token(xhashcash_challenge)headers = self.default_headers | {"X-Hashcash": xhashcash_token}- async with self.request_limiter, self.session.disabled():+ async with self._limiter, self.session.disabled():response = await self.session.post(self.entrypoint, params=params, json=data, timeout=self.timeout, headers=headers)diff --git a/cyberdrop_dl/managers/cache_manager.py b/cyberdrop_dl/managers/cache_manager.pyindex 1844a932c..875b5594f 100644--- a/cyberdrop_dl/managers/cache_manager.py+++ b/cyberdrop_dl/managers/cache_manager.py@@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Anyfrom aiohttp_client_cache import CacheBackend, SQLiteBackendfrom cyberdrop_dl import __version__ as current_version-from cyberdrop_dl.data_structures.supported_domains import SUPPORTED_FORUMS, SUPPORTED_WEBSITESfrom cyberdrop_dl.scraper.filters import cache_filter_fnfrom cyberdrop_dl.utils import yaml@@ -41,6 +40,8 @@ class CacheManager:self._cache = yaml.load(self.cache_file)def load_request_cache(self) -> None:+ from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_WEBSITES+rate_limiting_options = self.manager.config_manager.global_settings_data.rate_limiting_optionsurls_expire_after = {"*.simpcity.su": rate_limiting_options.file_host_cache_expire_after,diff --git a/cyberdrop_dl/data_structures/supported_domains.py b/cyberdrop_dl/supported_domains.pysimilarity index 100%rename from cyberdrop_dl/data_structures/supported_domains.pyrename to cyberdrop_dl/supported_domains.pydiff --git a/cyberdrop_dl/ui/prompts/user_prompts.py b/cyberdrop_dl/ui/prompts/user_prompts.pyindex 4e61d7847..dc66c1cf6 100644--- a/cyberdrop_dl/ui/prompts/user_prompts.py+++ b/cyberdrop_dl/ui/prompts/user_prompts.py@@ -16,11 +16,6 @@ from rich.console import Consolefrom cyberdrop_dl import __version__from cyberdrop_dl.constants import BROWSERS, RESERVED_CONFIG_NAMES-from cyberdrop_dl.data_structures.supported_domains import (- SUPPORTED_FORUMS,- SUPPORTED_SITES_DOMAINS,- SUPPORTED_WEBSITES,-)from cyberdrop_dl.ui.prompts import basic_promptsfrom cyberdrop_dl.ui.prompts.defaults import ALL_CHOICE, DONE_CHOICE, EXIT_CHOICEfrom cyberdrop_dl.utils.cookie_management import get_cookies_from_browsers@@ -160,6 +155,8 @@ class DomainType(IntEnum):def domains_prompt(*, domain_message: str = "Select site(s):") -> tuple[list[str], list[str]]:"""Asks the user to select website(s) for cookie actions and cache actions."""+ from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_SITES_DOMAINS, SUPPORTED_WEBSITES+OPTIONS = [["Forum", "File Host"], ["All Supported Websites"]]choices = basic_prompts.create_choices(OPTIONS)domain_type = basic_prompts.ask_choice(choices, message="Select category:")@@ -200,6 +197,7 @@ def domains_prompt(*, domain_message: str = "Select site(s):") -> tuple[list[strdef extract_cookies(manager: Manager, *, dry_run: bool = False) -> None:"""Asks the user to select browser(s) and domains(s) to import cookies from."""+ from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_SITES_DOMAINS, SUPPORTED_WEBSITESsupported_forums, supported_websites = list(SUPPORTED_FORUMS.values()), list(SUPPORTED_WEBSITES.values())domains, all_domains = domains_prompt(domain_message="Select site(s) to import cookies from:")diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.pyindex 5e6d5afc3..5e537dd4f 100644--- a/cyberdrop_dl/utils/cookie_management.py+++ b/cyberdrop_dl/utils/cookie_management.py@@ -21,12 +21,6 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Manager-from cyberdrop_dl.data_structures.supported_domains import (- SUPPORTED_FORUMS,- SUPPORTED_SITES_DOMAINS,- SUPPORTED_WEBSITES,-)-P = ParamSpec("P")R = TypeVar("R")@@ -77,6 +71,8 @@ def get_cookies_from_browsers(manager: Manager, *, browser: BROWSERS, domains: l:param domains: list of domains to filter cookies. If `None`, config `browser_cookies.sites` will be used:return: A set with all the domains that actually had cookies:raises BrowserCookieError: If there's any error while extracting cookies"""+ from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_SITES_DOMAINS, SUPPORTED_WEBSITES+if domains == []:msg = "No domains selected"raise ValueError(msg)@@ -164,7 +160,7 @@ async def read_netscape_files(cookie_files: list[Path]) -> AsyncIterable[tuple[slog(f"Cookies for {simplified_domain} are expired", 30)domains_seen.add(simplified_domain)- simple_cookie = _make_simple_cookie(cookie, now)+ simple_cookie = make_simple_cookie(cookie, now)yield cookie.domain, simple_cookie@@ -180,7 +176,7 @@ async def _read_netscape_file(file: Path) -> MozillaCookieJar | None:return await asyncio.to_thread(read)-def _make_simple_cookie(cookie: Cookie, now: float) -> SimpleCookie:+def make_simple_cookie(cookie: Cookie, now: float) -> SimpleCookie:simple_cookie = SimpleCookie()assert cookie.value is not Nonesimple_cookie[cookie.name] = cookie.valuediff --git a/tests/crawlers/test_cases/xhamster.py b/tests/crawlers/test_cases/xhamster.pyindex f5b758d3c..63d12906b 100644--- a/tests/crawlers/test_cases/xhamster.py+++ b/tests/crawlers/test_cases/xhamster.py@@ -56,7 +56,7 @@ TEST_CASES = ["datetime": 1372456260,}],- 42,+ 40,),("https://xhamster.com/creators/danielagomezz",diff --git a/tests/crawlers/test_cases/xvideos.py b/tests/crawlers/test_cases/xvideos.pyindex 1f468bb5c..69cb03e77 100644--- a/tests/crawlers/test_cases/xvideos.py+++ b/tests/crawlers/test_cases/xvideos.py@@ -52,7 +52,7 @@ TEST_CASES = [{"url": "https://it.xvideos.com/video.uceupff0647/_","filename": "Beauty Gina Gerson Takes Monster Cock in Anal - Anal Orgasm [uceupff0647][1080p].mp4",- "referer": "https://it.xvideos.com/video.uceupff0647/beauty_gina_gerson_takes_monster_cock_in_anal_-_anal_orgasm",+ "referer": "https://it.xvideos.com/video.uceupff0647/_",}],),