Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: update crawlers to use the new request method (#1281)

NTFSvolume8172948921e4
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
index d06cfd3a6..5f059ab76 100644
--- a/cyberdrop_dl/clients/flaresolverr.py
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -4,7 +4,6 @@ import asyncio
import dataclasses
import itertools
import time
-from functools import partial
from http.cookies import SimpleCookie
from typing import TYPE_CHECKING, Any
@@ -76,7 +75,7 @@ class FlareSolverr:
self.manager = manager
self._session_id: str = ""
self._session_lock, self._request_lock = asyncio.Lock(), asyncio.Lock()
- self._next_request_id: Callable[[], int] = partial(next, itertools.count(1))
+ self._next_request_id: Callable[[], int] = itertools.count(1).__next__
if manager.global_config.general.flaresolverr:
self.url = manager.global_config.general.flaresolverr / "v1"
else:
@@ -89,8 +88,6 @@ class FlareSolverr:
await self._destroy_session()
async def request(self, url: AbsoluteHttpURL, data: Any = None) -> FlareSolverrSolution:
- # TODO: make this method return an abstract response
-
invalid_response_error = DDOSGuardError("Invalid response from flaresolverr")
try:
if not self._session_id:
diff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.py
index d4e0a97f5..fc4b27d00 100644
--- a/cyberdrop_dl/clients/response.py
+++ b/cyberdrop_dl/clients/response.py
@@ -3,12 +3,16 @@ from __future__ import annotations
import asyncio
import dataclasses
from json import loads as json_loads
+from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Self
+import aiohttp.multipart
from aiohttp import ClientResponse
+from aiohttp.client_reqrep import ContentDisposition
from aiohttp_client_cache.response import CachedResponse
from bs4 import BeautifulSoup
from multidict import CIMultiDict, CIMultiDictProxy
+from propcache import under_cached_property
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import InvalidContentTypeError, ScrapeError
@@ -22,10 +26,12 @@ if TYPE_CHECKING:
@dataclasses.dataclass(slots=True, weakref_slot=True)
class AbstractResponse:
- """Class to represent common methods and attributes between:
- - `aiohttp.ClientResponse`
- - `curl_cffi.Response`
- - `FlareSolverrSolution`"""
+ """
+ Class to represent common methods and attributes between:
+ - `aiohttp.ClientResponse`
+ - `curl_cffi.Response`
+ - `FlareSolverrSolution`
+ """
content_type: str
status: int
@@ -35,6 +41,7 @@ class AbstractResponse:
_resp: ClientResponse | CachedResponse | CurlResponse | None = None
_text: str = ""
+ _cache: dict[str, Any] = dataclasses.field(init=False, default_factory=dict)
_read_lock: asyncio.Lock = dataclasses.field(init=False, default_factory=asyncio.Lock)
@classmethod
@@ -44,7 +51,8 @@ class AbstractResponse:
headers = response.headers
else:
status = response.status_code
- headers = CIMultiDictProxy(CIMultiDict({k: v or "" for k, v in response.headers}))
+ multi_items = ((k, v) for k, v in response.headers.multi_items() if v is not None)
+ headers = CIMultiDictProxy(CIMultiDict(multi_items))
url = AbsoluteHttpURL(response.url)
content_type, location = cls.parse_headers(url, headers)
@@ -70,7 +78,7 @@ class AbstractResponse:
)
@staticmethod
- def parse_headers(url: AbsoluteHttpURL, headers: CIMultiDictProxy[str]):
+ def parse_headers(url: AbsoluteHttpURL, headers: CIMultiDictProxy[str]) -> tuple[str, AbsoluteHttpURL | None]:
if location := headers.get("location"):
location = parse_url(location, url.origin(), trim=False)
else:
@@ -79,6 +87,38 @@ class AbstractResponse:
content_type = (headers.get("Content-Type") or "").lower()
return content_type, location
+ @under_cached_property
+ def content_disposition(self) -> ContentDisposition:
+ header = self.headers["Content-Disposition"]
+ disposition_type, params = aiohttp.multipart.parse_content_disposition(header)
+ params = MappingProxyType(params)
+ filename = aiohttp.multipart.content_disposition_filename(params)
+ return ContentDisposition(disposition_type, params, filename)
+
+ @property
+ def filename(self) -> str:
+ assert self.content_disposition.filename
+ return self.content_disposition.filename
+
+ @property
+ def consumed(self) -> bool:
+ return bool(self._text)
+
+ @property
+ def ok(self) -> bool:
+ """Returns `True` if `status` is less than `400`, `False` if not.
+
+ This is **not** a check for ``200 OK``
+ """
+ return self.status < 400
+
+ async def read(self) -> bytes:
+ assert self._resp is not None
+ async with self._read_lock:
+ if isinstance(self._resp, ClientResponse | CachedResponse):
+ return await self._resp.read()
+ return await self._resp.acontent()
+
async def text(self, encoding: str | None = None) -> str:
if self._text:
return self._text
@@ -119,4 +159,4 @@ class AbstractResponse:
raise InvalidContentTypeError(message=msg)
def __repr__(self) -> str:
- return f"<{self.__class__.__name__} [{self.status}] ({self.url!r})>"
+ return f"<{self.__class__.__name__} [{self.status}] ({self.url})>"
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
index e00721edf..0c49e1c94 100644
--- a/cyberdrop_dl/clients/scraper_client.py
+++ b/cyberdrop_dl/clients/scraper_client.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import contextlib
+import time
from datetime import datetime
from json import dumps as json_dumps
from pathlib import Path
@@ -10,6 +11,7 @@ from typing import TYPE_CHECKING, Any, cast
import cyberdrop_dl.constants as constants
from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.exceptions import DDOSGuardError
+from cyberdrop_dl.utils.cookie_management import make_simple_cookie
from cyberdrop_dl.utils.utilities import sanitize_filename
if TYPE_CHECKING:
@@ -78,6 +80,19 @@ class ScraperClient:
finally:
await self.write_soup_to_disk(url, resp, exc)
+ def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:
+ """
+ Apply to the cookies from the `curl` session into the `aiohttp` session, filtering them by the URL
+
+ This is mostly just to get the `cf_cleareance` cookie value into the `aiohttp` session
+
+ The reverse (sync `aiohttp` -> `curl`) is not needed at the moment, so it is skipped
+ """
+ now = time.time()
+ for cookie in self.client_manager._curl_session.cookies.jar:
+ simple_cookie = make_simple_cookie(cookie, now)
+ self.client_manager.cookies.update_cookies(simple_cookie, url)
+
@contextlib.asynccontextmanager
async def __request_context(
self,
@@ -95,8 +110,7 @@ class ScraperClient:
curl_resp = await self.client_manager._curl_session.request(method, str(url), stream=True, **request_params)
try:
yield AbstractResponse.from_resp(curl_resp)
- curl_cookies = self.client_manager._curl_session.cookies.get_dict(url.host)
- self.client_manager.cookies.update_cookies(curl_cookies, url)
+ self.__sync_session_cookies(url)
finally:
await curl_resp.aclose()
return
@@ -121,15 +135,14 @@ class ScraperClient:
async def write_soup_to_disk(self, url: AbsoluteHttpURL, response: AbstractResponse, exc: Exception | None = None):
"""Writes html to a file."""
- if not self._save_pages_html:
+ if not (
+ self._save_pages_html
+ and "html" in response.content_type
+ and response.consumed # Do not consume the response if the crawler didn't
+ ):
return
- content: str = await response.text()
- try:
- content = cast("str", (await response.soup()).prettify(formatter="html"))
- except Exception:
- pass
-
+ content = cast("str", (await response.soup()).prettify(formatter="html"))
now = datetime.now()
log_date = now.strftime(constants.LOGS_DATETIME_FORMAT)
url_str = str(url)
diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.py
index a062d8c80..d8a98d283 100755
--- a/cyberdrop_dl/config/config_model.py
+++ b/cyberdrop_dl/config/config_model.py
@@ -9,7 +9,6 @@ from pydantic import BaseModel, ByteSize, NonNegativeInt, PositiveInt, field_ser
from cyberdrop_dl import constants
from cyberdrop_dl.constants import BROWSERS, DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGE
from cyberdrop_dl.data_structures.hash import Hashing
-from cyberdrop_dl.data_structures.supported_domains import SUPPORTED_SITES_DOMAINS
from cyberdrop_dl.models import HttpAppriseURL
from cyberdrop_dl.models.types import (
ByteSizeSerilized,
@@ -22,6 +21,7 @@ from cyberdrop_dl.models.types import (
PathOrNone,
)
from cyberdrop_dl.models.validators import falsy_as, to_timedelta
+from cyberdrop_dl.supported_domains import SUPPORTED_SITES_DOMAINS
from cyberdrop_dl.utils.strings import validate_format_string
from cyberdrop_dl.utils.utilities import purge_dir_tree
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index cebbb1efc..d8c01c476 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -149,7 +149,9 @@ class CheveretoCrawler(Crawler, is_generic=True):
raise PasswordProtectedError
soup = await self.request_soup(
- _sort_by_new(scrape_item.url / ""), method="POST", data={"content-password": password}
+ _sort_by_new(scrape_item.url / ""),
+ method="POST",
+ data={"content-password": password},
)
if _is_password_protected(soup):
@@ -167,9 +169,7 @@ class CheveretoCrawler(Crawler, is_generic=True):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup = await self.request_soup(scrape_item.url)
-
+ soup = await self.request_soup(scrape_item.url)
link_str = open_graph.get("video", soup) or open_graph.get("image", soup)
if not link_str or "loading.svg" in link_str:
link_str = Selector.MAIN_IMAGE(soup)
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index bd49b4d18..64c80a032 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -25,7 +25,7 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import LoginError, MaxChildrenError, ScrapeError
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.dates import TimeStamp, to_timestamp
-from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, is_absolute_http_url, is_blob_or_svg
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, is_blob_or_svg
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Iterable, Sequence
@@ -178,17 +178,17 @@ class MessageBoardCrawler(Crawler, is_abc=True):
def parse_thread(cls, url: AbsoluteHttpURL, thread_name_and_id: str) -> ThreadProtocol: ...
@abstractmethod
- async def post(self, scrape_item: ScrapeItem, post: ForumPostProtocol) -> None: ...
+ async def post(self, scrape_item: ScrapeItem, /, post: ForumPostProtocol) -> None: ...
@abstractmethod
- async def thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None: ...
+ async def thread(self, scrape_item: ScrapeItem, /, thread: ThreadProtocol) -> None: ...
async def forum(self, scrape_item: ScrapeItem) -> None:
# Subclasses can define custom logic for this method.
# They would need to also override `fetch` since the default fetch does not take this method into account
raise NotImplementedError
- async def resolve_confirmation_link(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
+ async def resolve_confirmation_link(self, url: AbsoluteHttpURL, /) -> AbsoluteHttpURL | None:
# Not every forum has confirmation link so overriding this method is optional
# Implementation of this method MUST return `None` instead of raising an error
raise NotImplementedError
@@ -251,27 +251,6 @@ class MessageBoardCrawler(Crawler, is_abc=True):
by_host = any(host in link.host for host in self.ATTACHMENT_HOSTS)
return by_parts or by_host
- # TODO: Move this to base crawler
- @final
- @error_handling_wrapper
- async def follow_redirect_w_get(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- response, _ = await self.client._get_response_and_soup(self.DOMAIN, scrape_item.url)
- assert is_absolute_http_url(response.url)
- scrape_item.url = response.url
- self.manager.task_group.create_task(self.run(scrape_item))
-
- # TODO: Move this to base crawler
- @final
- @error_handling_wrapper
- async def follow_redirect_w_head(self, scrape_item: ScrapeItem) -> None:
- head = await self.client.get_head(self.DOMAIN, scrape_item.url)
- redirect = head.get("location")
- if not redirect:
- raise ScrapeError(422)
- scrape_item.url = self.parse_url(redirect)
- self.manager.task_group.create_task(self.run(scrape_item))
-
@final
async def follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:
url = await self.resolve_confirmation_link(scrape_item.url)
@@ -310,7 +289,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):
if self.is_attachment(link):
return await self.handle_internal_link(scrape_item, link)
if self.PRIMARY_URL.host == link.host:
- self.manager.task_group.create_task(self.run(scrape_item.create_child(link)))
+ self.create_task(self.run(scrape_item.create_child(link)))
return
new_scrape_item = scrape_item.create_child(link)
self.handle_external_links(new_scrape_item)
@@ -353,8 +332,9 @@ class MessageBoardCrawler(Crawler, is_abc=True):
self.log(msg, 30)
async def check_login_with_request(self, login_url: AbsoluteHttpURL) -> tuple[str, bool]:
- text = await self.client.get_text(self.DOMAIN, login_url, cache_disabled=True)
- return text, any(p in text for p in ('<span class="p-navgroup-user-linkText">', "You are already logged in."))
+ text = await self.request_text(login_url, cache_disabled=True)
+ logged_in = '<span class="p-navgroup-user-linkText">' in text or "You are already logged in." in text
+ return text, logged_in
class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
@@ -417,7 +397,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
return thread.url / f"{cls.POST_URL_PART_NAME}-{post_id}"
@error_handling_wrapper
- async def thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None:
+ async def thread(self, scrape_item: ScrapeItem, /, thread: ThreadProtocol) -> None:
scrape_item.setup_as_forum("")
if thread.url in self.scraped_threads:
return
@@ -465,7 +445,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
possible_datetime=current_post.timestamp,
add_parent=post_url,
)
- self.manager.task_group.create_task(self.post(new_scrape_item, current_post))
+ self.create_task(self.post(new_scrape_item, current_post))
try:
scrape_item.add_children()
except MaxChildrenError:
@@ -582,12 +562,12 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
if url.startswith("https://"):
return self.parse_url(url)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, link)
+ soup = await self.request_soup(link)
selector = self.SELECTORS.confirmation_button
confirm_button = soup.select_one(selector.element)
if not confirm_button:
return
+
link_str: str = css.get_attr(confirm_button, selector.attribute)
link_str = link_str.split('" class="link link--internal', 1)[0]
new_link = self.parse_url(link_str)
diff --git a/cyberdrop_dl/crawlers/_kemono_base.py b/cyberdrop_dl/crawlers/_kemono_base.py
index 11a4789bb..572c716fa 100644
--- a/cyberdrop_dl/crawlers/_kemono_base.py
+++ b/cyberdrop_dl/crawlers/_kemono_base.py
@@ -283,7 +283,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
for channel in server.channels:
url = self.PRIMARY_URL / "discord/server" / server_id / channel.id
new_scrape_item = scrape_item.create_child(url)
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
async def discord_channel(self, scrape_item: ScrapeItem, channel_id: str) -> None:
@@ -383,7 +383,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
url = self.PRIMARY_URL / service / "user" / user_id
new_scrape_item = scrape_item.create_child(url)
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
# ~~~~~~~~ INTERNAL METHODS, not expected to be overriden, but could be ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.py
index 360f945ae..08b0999cd 100644
--- a/cyberdrop_dl/crawlers/_kvs.py
+++ b/cyberdrop_dl/crawlers/_kvs.py
@@ -5,8 +5,6 @@ from __future__ import annotations
import re
from typing import TYPE_CHECKING, ClassVar, NamedTuple
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
@@ -60,9 +58,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
}
NEXT_PAGE_SELECTOR: ClassVar[str] = _SELECTORS.NEXT_PAGE
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(3, 10)
+ _RATE_LIMIT = 3, 10
async def fetch(self, scrape_item: ScrapeItem) -> None:
if any(p in scrape_item.url.parts for p in ("categories", "tags")) or scrape_item.url.query.get("q"):
@@ -79,8 +75,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def search(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
title = ""
if (search_query := scrape_item.url.query.get("q")) or scrape_item.url.parts[1] == "search":
search_query = search_query or scrape_item.url.parts[2]
@@ -100,8 +95,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def profile(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
user_name: str = css.select_one_get_text(soup, _SELECTORS.USER_NAME).split("'s Profile")[0].strip()
title = f"{user_name} [user]"
@@ -119,16 +113,14 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
url = scrape_item.url / video_category if video_category else scrape_item.url
async for soup in self.web_pager(url):
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEOS):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
-
+ soup = await self.request_soup(scrape_item.url)
video = self.get_video_info(soup)
filename, ext = self.get_filename_and_ext(video.url.name)
custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=video.res)
@@ -140,25 +132,22 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
-
- js_text: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_ID)
- album_id: str = get_text_between(js_text, "params['album_id'] =", ";").strip()
+ soup = await self.request_soup(scrape_item.url)
+ js_text = css.select_one_get_text(soup, _SELECTORS.ALBUM_ID)
+ album_id = get_text_between(js_text, "params['album_id'] =", ";").strip()
results = await self.get_album_results(album_id)
- title: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_NAME)
+ title = css.select_one_get_text(soup, _SELECTORS.ALBUM_NAME)
title = self.create_title(f"{title} [album]", album_id)
scrape_item.setup_as_album(title, album_id=album_id)
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUM_PICTURES, results=results):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def picture(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
url = self.parse_url(css.select_one_get_attr(soup, _SELECTORS.PICTURE, "src"))
filename, ext = self.get_filename_and_ext(url.name)
await self.handle_file(url, scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/_one_manager.py b/cyberdrop_dl/crawlers/_one_manager.py
index cd8e645da..b8c012143 100644
--- a/cyberdrop_dl/crawlers/_one_manager.py
+++ b/cyberdrop_dl/crawlers/_one_manager.py
@@ -50,8 +50,7 @@ class OneManagerCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def process_path(self, scrape_item: ScrapeItem) -> None:
try:
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
except InvalidContentTypeError: # This is a file, not html
scrape_item.parent_title = scrape_item.parent_title.rsplit("/", 1)[0]
link = scrape_item.url
@@ -71,7 +70,7 @@ class OneManagerCrawler(Crawler, is_abc=True):
for folder in css.iselect(table, _SELECTORS.FOLDER):
link = scrape_item.url / css.select_one_get_attr(folder, _SELECTORS.FOLDER_LINK, "href")
new_scrape_item = scrape_item.create_child(link, new_title_part=link.name)
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
@error_handling_wrapper
diff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.py
index 3b84c59c9..05c64aaa3 100644
--- a/cyberdrop_dl/crawlers/_yetishare.py
+++ b/cyberdrop_dl/crawlers/_yetishare.py
@@ -3,7 +3,6 @@ from __future__ import annotations
import itertools
from typing import TYPE_CHECKING, Any, ClassVar
-from aiolimiter import AsyncLimiter
from bs4 import BeautifulSoup
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id
@@ -46,6 +45,7 @@ class YetiShareCrawler(Crawler, is_abc=True):
),
"Shared folders": "/shared/<share_key>",
}
+ _RATE_LIMIT = 5, 1
def __init_subclass__(cls, **kwargs) -> None:
super().__init_subclass__(**kwargs)
@@ -53,9 +53,6 @@ class YetiShareCrawler(Crawler, is_abc=True):
cls.FILE_API_URL = cls.PRIMARY_URL / "account/ajax/file_details"
cls.FOLDER_PASSWORD_API_URL = cls.PRIMARY_URL / "ajax/folder_password_process"
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(5, 1)
-
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["folder", folder_id, *_]:
@@ -72,8 +69,7 @@ class YetiShareCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def folder(self, scrape_item: ScrapeItem, folder_id: str, is_shared: bool = False) -> None:
# Make request to update cookies. Access to folders is saved in cookies
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
if soup.select_one(Selector.LOGIN_FORM):
raise ScrapeError(410, "Folder has been deleted")
@@ -133,9 +129,7 @@ class YetiShareCrawler(Crawler, is_abc=True):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
-
+ soup = await self.request_soup(scrape_item.url)
if soup.select_one(Selector.PASSWORD_PROTECTED):
soup = await self._unlock_password_protected_file(scrape_item, file_id)
@@ -168,7 +162,8 @@ class YetiShareCrawler(Crawler, is_abc=True):
link = self.parse_url(raw_link).with_query(download_token=token)
scrape_item.possible_datetime = self.parse_date(
- css.select_one_get_text(soup, Selector.FILE_UPLOAD_DATE), "%d/%m/%Y %H:%M:%S"
+ css.select_one_get_text(soup, Selector.FILE_UPLOAD_DATE),
+ "%d/%m/%Y %H:%M:%S",
)
filename = css.select_one_get_text(soup, Selector.FILE_NAME)
@@ -182,13 +177,12 @@ class YetiShareCrawler(Crawler, is_abc=True):
) -> BeautifulSoup:
async def ajax_api_request() -> BeautifulSoup:
ajax_url = self.FILE_API_URL if is_file else self.FOLDERS_API_URL
- async with self.request_limiter:
- json_resp: dict[str, str] = await self.client.post_data(
- self.DOMAIN,
- ajax_url,
- data=data,
- headers={"X-Requested-With": "XMLHttpRequest"},
- )
+ json_resp: dict[str, str] = await self.request_json(
+ ajax_url,
+ method="POST",
+ data=data,
+ headers={"X-Requested-With": "XMLHttpRequest"},
+ )
return BeautifulSoup(json_resp["html"].replace("\\", ""), "html.parser")
@@ -207,13 +201,16 @@ class YetiShareCrawler(Crawler, is_abc=True):
raise PasswordProtectedError
password_post_url = (self.PRIMARY_URL / file_id).with_query("pt=")
- async with self.request_limiter:
- resp_bytes = await self.client.post_data_raw(
- self.DOMAIN,
- password_post_url,
- data={"filePassword": password, "submitme": 1},
- )
- soup = BeautifulSoup(resp_bytes, "html.parser")
+
+ content = await self.request_text(
+ password_post_url,
+ method="POST",
+ data={
+ "filePassword": password,
+ "submitme": 1,
+ },
+ )
+ soup = BeautifulSoup(content, "html.parser")
if soup.select_one(Selector.PASSWORD_PROTECTED):
raise PasswordProtectedError("File password is invalid")
@@ -225,16 +222,15 @@ class YetiShareCrawler(Crawler, is_abc=True):
raise PasswordProtectedError
# Make a request with the password. Access to the file/folder will be stored in cookies
- async with self.request_limiter:
- json_resp: dict = await self.client.post_data(
- self.DOMAIN,
- self.FOLDER_PASSWORD_API_URL,
- data={
- "folderPassword": password,
- "folderId": node_id,
- "submitme": 1,
- },
- )
+ json_resp: dict[str, Any] = await self.request_json(
+ self.FOLDER_PASSWORD_API_URL,
+ method="POST",
+ data={
+ "folderPassword": password,
+ "folderId": node_id,
+ "submitme": 1,
+ },
+ )
if not json_resp.get("success"):
raise PasswordProtectedError(message="Incorrect password")
diff --git a/cyberdrop_dl/crawlers/archivebate.py b/cyberdrop_dl/crawlers/archivebate.py
index b558f1a7d..53df85361 100644
--- a/cyberdrop_dl/crawlers/archivebate.py
+++ b/cyberdrop_dl/crawlers/archivebate.py
@@ -2,8 +2,6 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.mixdrop import MixDropCrawler
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
@@ -11,8 +9,6 @@ from cyberdrop_dl.utils import css, open_graph
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.crawlers.crawler import SupportedDomains, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -37,9 +33,7 @@ class ArchiveBateCrawler(MixDropCrawler):
FOLDER_DOMAIN: ClassVar[str] = "ArchiveBate"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
NEXT_PAGE_SELECTOR = _SELECTORS.NEXT_PAGE
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(4, 1)
+ _RATE_LIMIT = 4, 1
async def fetch(self, scrape_item: ScrapeItem) -> None:
if "watch" in scrape_item.url.parts:
@@ -68,8 +62,7 @@ class ArchiveBateCrawler(MixDropCrawler):
self.manager.progress_manager.download_progress.add_previously_completed()
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
if "This video has been deleted" in soup.text:
raise ScrapeError(410)
@@ -88,8 +81,7 @@ class ArchiveBateCrawler(MixDropCrawler):
if await self.check_complete_from_referer(mixdrop_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, mixdrop_url)
+ soup = await self.request_soup(mixdrop_url)
link = self.create_download_link(soup)
filename, ext = self.get_filename_and_ext(link.name)
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.py
index 5c71f6d83..cd7aeaf6b 100644
--- a/cyberdrop_dl/crawlers/ashemaletube.py
+++ b/cyberdrop_dl/crawlers/ashemaletube.py
@@ -4,8 +4,6 @@ import json
from enum import StrEnum
from typing import TYPE_CHECKING, ClassVar, NamedTuple
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
@@ -81,9 +79,7 @@ class AShemaleTubeCrawler(Crawler):
FOLDER_DOMAIN: ClassVar[str] = "aShemaleTube"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
NEXT_PAGE_SELECTOR: ClassVar[str] = _SELECTORS.NEXT_PAGE
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(3, 10)
+ _RATE_LIMIT = 3, 10
async def fetch(self, scrape_item: ScrapeItem) -> None:
if any(p in scrape_item.url.parts for p in ("creators", "profiles", "pornstars", "model")):
@@ -108,7 +104,7 @@ class AShemaleTubeCrawler(Crawler):
async def gallery(self, scrape_item: ScrapeItem) -> None:
async for soup in self.web_pager(scrape_item.url, cffi=True):
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.GALLERY_ALBUM):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
@@ -132,7 +128,7 @@ class AShemaleTubeCrawler(Crawler):
else:
scrape_item.setup_as_album(collection_title)
for _, new_scrape_item in self.iter_children(scrape_item, soup, MEDIA_SELECTOR_MAP[collection_type]):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
def create_collection_title(self, soup: BeautifulSoup, collection_type: CollectionType) -> str:
title_elem = soup.select_one(TITLE_SELECTOR_MAP[collection_type])
@@ -147,8 +143,7 @@ class AShemaleTubeCrawler(Crawler):
async def image(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item.url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url, impersonate=True)
img_item = soup.select_one(_SELECTORS.IMAGE_ITEM)
if not img_item:
raise ScrapeError(404)
@@ -173,8 +168,7 @@ class AShemaleTubeCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url, impersonate=True)
if soup.select_one(_SELECTORS.LOGIN_REQUIRED):
raise ScrapeError(401)
diff --git a/cyberdrop_dl/crawlers/beeg.py b/cyberdrop_dl/crawlers/beeg.py
index fcd6dd4c0..c7bf20b7e 100644
--- a/cyberdrop_dl/crawlers/beeg.py
+++ b/cyberdrop_dl/crawlers/beeg.py
@@ -2,8 +2,6 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -32,24 +30,20 @@ class BeegComCrawler(Crawler):
}
DOMAIN: ClassVar[str] = "beeg.com"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
+ _RATE_LIMIT = 4, 1
async def fetch(self, scrape_item: ScrapeItem) -> None:
if video_id := get_video_id(scrape_item.url):
return await self.video(scrape_item, video_id)
raise ValueError
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(4, 1)
-
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
canonical_url = PRIMARY_URL / video_id
if await self.check_complete_from_referer(canonical_url):
return
scrape_item.url = canonical_url
- async with self.request_limiter:
- json_resp: dict[str, Any] = await self.client.get_json(self.DOMAIN, JSON_URL / video_id)
-
+ json_resp: dict[str, Any] = await self.request_json(JSON_URL / video_id)
facts: dict[str, Any] = min(json_resp["fc_facts"], key=lambda x: int(x["id"]))
file: dict[str, Any] = json_resp["file"]
title: str = next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"]
diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.py
index dad94b496..30dc4bc04 100644
--- a/cyberdrop_dl/crawlers/box_dot_com.py
+++ b/cyberdrop_dl/crawlers/box_dot_com.py
@@ -16,8 +16,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -75,8 +73,7 @@ class BoxDotComCrawler(Crawler):
if "file" in scrape_item.url.parts and await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
if "file or folder link has been removed" in soup.text:
raise ScrapeError(410)
@@ -97,7 +94,7 @@ class BoxDotComCrawler(Crawler):
_, file_id = file_key.rsplit("f_", 1)
canonical_url = get_canonical_url(shared_name, file_id)
scrape_item.url = canonical_url
- self.manager.task_group.create_task(self.run(scrape_item))
+ self.create_task(self.run(scrape_item))
return
shared_folder = SharedFolder(**shared_folder_data)
diff --git a/cyberdrop_dl/crawlers/bunkrr_albums_io.py b/cyberdrop_dl/crawlers/bunkrr_albums_io.py
index 9bd10b712..e61ea6e6a 100644
--- a/cyberdrop_dl/crawlers/bunkrr_albums_io.py
+++ b/cyberdrop_dl/crawlers/bunkrr_albums_io.py
@@ -37,17 +37,16 @@ class BunkrAlbumsIOCrawler(Crawler):
search_query = scrape_item.url.query["search"]
title = self.create_title(search_query)
scrape_item.setup_as_profile(title)
- async for soup in self.web_pager(scrape_item):
+ async for soup in self._pager(scrape_item):
for _, new_scrape_item in self.iter_children(scrape_item, soup, ALBUM_SELECTOR):
self.handle_external_links(new_scrape_item)
- async def web_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:
+ async def _pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:
"""Generator of website pages."""
page_url = scrape_item.url
while True:
current_page_number = int(page_url.query.get("page") or 1)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, page_url)
+ soup = await self.request_soup(page_url)
next_page = soup.select(self.NEXT_PAGE_SELECTOR)
yield soup
if not next_page:
diff --git a/cyberdrop_dl/crawlers/buzzheavier.py b/cyberdrop_dl/crawlers/buzzheavier.py
index c3858dab5..aa0b2229d 100644
--- a/cyberdrop_dl/crawlers/buzzheavier.py
+++ b/cyberdrop_dl/crawlers/buzzheavier.py
@@ -27,12 +27,16 @@ class BuzzHeavierCrawler(Crawler):
return
url = scrape_item.url / "download"
- headers = {"HX-Current-URL": str(scrape_item.url), "HX-Request": "true"}
- async with self.request_limiter:
- response = await self.client._get_head(self.DOMAIN, url, headers=headers)
-
- assert response.content_disposition
- assert response.content_disposition.filename
- link = self.parse_url(response.headers["hx-redirect"])
- filename, ext = self.get_filename_and_ext(response.content_disposition.filename, assume_ext=".zip")
+ async with self.request(
+ url,
+ method="HEAD",
+ headers={
+ "HX-Current-URL": str(scrape_item.url),
+ "HX-Request": "true",
+ },
+ ) as resp:
+ filename: str = resp.filename
+
+ link = self.parse_url(resp.headers["hx-redirect"])
+ filename, ext = self.get_filename_and_ext(filename, assume_ext=".zip")
await self.handle_file(scrape_item.url, scrape_item, filename, ext, debrid_link=link)
diff --git a/cyberdrop_dl/crawlers/camwhores_dot_tv.py b/cyberdrop_dl/crawlers/camwhores_dot_tv.py
index a559fae16..ef03d8387 100644
--- a/cyberdrop_dl/crawlers/camwhores_dot_tv.py
+++ b/cyberdrop_dl/crawlers/camwhores_dot_tv.py
@@ -40,7 +40,7 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler):
return video._replace(title=open_graph.title(soup))
def parse_url(
- self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = False
+ self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool | None = None
) -> AbsoluteHttpURL:
return super().parse_url(link_str, relative_to, trim=False)
@@ -57,8 +57,7 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler):
async def iter_videos(self, scrape_item: ScrapeItem, video_category: str = "") -> None:
url = scrape_item.url / video_category if video_category else scrape_item.url
await super().iter_videos(scrape_item, video_category)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, url)
+ soup = await self.request_soup(url)
last_page = int(css.get_text(soup.select(LAST_PAGE_SELECTOR)[-1]))
# TODO: Porntrex also uses KVS. Make the KVS crawler handle it by default
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 86db0caf1..7c7006775 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -270,14 +270,16 @@ class Crawler(ABC):
try:
yield task_id
except ValueError:
- await self.raise_e(scrape_item, ScrapeError("Unknown URL path"))
+ self.raise_exc(scrape_item, ScrapeError("Unknown URL path"))
except MaxChildrenError as e:
- await self.raise_e(scrape_item, e)
+ self.raise_exc(scrape_item, e)
finally:
pass
@error_handling_wrapper
- def raise_e(self, scrape_item: ScrapeItem, exc: type[Exception] | Exception) -> None:
+ def raise_exc(self, scrape_item: ScrapeItem, exc: type[Exception] | Exception | str) -> None:
+ if isinstance(exc, str):
+ exc = ScrapeError(exc)
raise exc
@final
@@ -427,7 +429,7 @@ class Crawler(ABC):
def handle_external_links(self, scrape_item: ScrapeItem) -> None:
"""Maps external links to the scraper class."""
scrape_item.reset()
- self.manager.task_group.create_task(self.manager.scrape_mapper.filter_and_send_to_crawler(scrape_item))
+ self.create_task(self.manager.scrape_mapper.filter_and_send_to_crawler(scrape_item))
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -660,6 +662,24 @@ class Crawler(ABC):
async with self.request(url, method="HEAD") as head:
return head.location or url
+ @final
+ @error_handling_wrapper
+ async def follow_redirect_w_get(self, scrape_item: ScrapeItem) -> None:
+ async with self.request(scrape_item.url) as resp:
+ if resp.url == scrape_item.url:
+ raise ScrapeError(422)
+ scrape_item.url = resp.url
+ self.create_task(self.run(scrape_item))
+
+ @final
+ @error_handling_wrapper
+ async def follow_redirect_w_head(self, scrape_item: ScrapeItem) -> None:
+ location = await self._get_redirect_url(scrape_item.url)
+ if location == scrape_item.url:
+ raise ScrapeError(422)
+ scrape_item.url = location
+ self.create_task(self.run(scrape_item))
+
@staticmethod
def register_cache_filter(
url: URL, filter_fn: Callable[[AnyResponse], bool] | Callable[[AnyResponse], Awaitable[bool]]
diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.py
index 59825576e..81c91d77e 100644
--- a/cyberdrop_dl/crawlers/cyberdrop.py
+++ b/cyberdrop_dl/crawlers/cyberdrop.py
@@ -1,9 +1,7 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
-
-from aiolimiter import AsyncLimiter
-from bs4 import BeautifulSoup
+import asyncio
+from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -12,8 +10,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
API_ENTRYPOINT = AbsoluteHttpURL("https://api.cyberdrop.me/api/")
@@ -40,9 +36,6 @@ class CyberdropCrawler(Crawler):
_RATE_LIMIT: ClassVar[tuple[float, float]] = 5, 1
_DOWNLOAD_SLOTS: ClassVar[int | None] = 1
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(1, 2)
-
async def fetch(self, scrape_item: ScrapeItem) -> None:
if "a" in scrape_item.url.parts:
return await self.album(scrape_item)
@@ -53,8 +46,7 @@ class CyberdropCrawler(Crawler):
scrape_item.url = scrape_item.url.with_query("nojs")
album_id = scrape_item.url.parts[2]
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
try:
title: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_TITLE)
@@ -68,36 +60,32 @@ class CyberdropCrawler(Crawler):
scrape_item.possible_datetime = self.parse_date(date_tags[-1].text, "%d.%m.%Y")
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUM_ITEM):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
- scrape_item.url = await self.get_stream_link(scrape_item.url)
+ scrape_item.url = await self._get_stream_link(scrape_item.url)
if await self.check_complete_from_referer(scrape_item):
return
file_id = scrape_item.url.name
- async with self.request_limiter:
- api_url = API_ENTRYPOINT / "file" / "info" / file_id
- json_resp = await self.client.get_json(self.DOMAIN, api_url)
-
- filename, ext = self.get_filename_and_ext(json_resp["name"])
+ file_info: tuple[dict[str, Any], dict[str, Any]] = await asyncio.gather(
+ self.request_json(API_ENTRYPOINT / "file" / "info" / file_id),
+ self.request_json(API_ENTRYPOINT / "file" / "auth" / file_id),
+ )
- async with self.request_limiter:
- api_url = API_ENTRYPOINT / "file" / "auth" / file_id
- json_resp = await self.client.get_json(self.DOMAIN, api_url)
-
- link = self.parse_url(json_resp["url"])
+ filename, ext = self.get_filename_and_ext(file_info[0]["name"])
+ link = self.parse_url(file_info[1]["url"])
await self.handle_file(link, scrape_item, filename, ext)
- async def get_stream_link(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- """Gets the stream link for a given URL.
-
- NOTE: This makes a request to get the final URL (if necessary). Calling function must use `@error_handling_wrapper`"""
+ async def _get_stream_link(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ """Gets the stream link for a given URL."""
if any(part in url.parts for part in ("a", "f")):
return url
+
if url.host.count(".") > 1 or "e" in url.parts:
return PRIMARY_URL / "f" / url.name
- response, _ = await self.client._get_response_and_soup(self.DOMAIN, url)
- return AbsoluteHttpURL(response.url)
+
+ async with self.request(url) as resp:
+ return resp.url
diff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.py
index 661b56fa7..66bb46857 100644
--- a/cyberdrop_dl/crawlers/dirtyship.py
+++ b/cyberdrop_dl/crawlers/dirtyship.py
@@ -66,8 +66,7 @@ class DirtyShipCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
if not scrape_item.url.suffix == ".jpg":
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
url = self.parse_url(
next(css.get_attr(a, "href") for a in soup.select(_SELECTORS.SINGLE_PHOTO) if "full" in a.get_text())
)
@@ -118,12 +117,11 @@ class DirtyShipCrawler(Crawler):
scrape_item.setup_as_album(title)
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.PLAYLIST_ITEM):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
title: str = css.select_one_get_text(soup, "title")
title = title.split(" - DirtyShip")[0]
diff --git a/cyberdrop_dl/crawlers/discourse/_discourse.py b/cyberdrop_dl/crawlers/discourse/_discourse.py
index 7493038e4..b318f0c5e 100644
--- a/cyberdrop_dl/crawlers/discourse/_discourse.py
+++ b/cyberdrop_dl/crawlers/discourse/_discourse.py
@@ -65,8 +65,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
async def make_request(self, model_cls: type[_ModelT], path: str, params: dict[str, Any] | None = None) -> _ModelT:
api_url = self.PRIMARY_URL.joinpath(path)
- async with self.request_limiter:
- json_text = await self.client.get_text(self.DOMAIN, api_url, request_params={"params": params})
+ json_text = await self.request_text(api_url, params=params)
return model_cls.model_validate_json(json_text, by_alias=True, by_name=True)
@error_handling_wrapper
@@ -87,13 +86,13 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
await self.handle_link(scrape_item, self.parse_url(topic.image_url))
await self.process_posts(scrape_item, topic)
- async def forum(self, scrape_item: ScrapeItem) -> None:
+ async def forum(self, scrape_item: ScrapeItem, /) -> None:
raise NotImplementedError
def parse_thread(self, *args, **kwargs) -> None:
raise NotImplementedError
- async def thread(self, scrape_item: ScrapeItem, thread: Topic) -> None:
+ async def thread(self, scrape_item: ScrapeItem, /, thread: Topic) -> None:
await self.topic(scrape_item, thread)
@error_handling_wrapper
@@ -127,7 +126,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
if topic.init_post_number != 1 and self.scrape_single_forum_post:
return
- async def post(self, scrape_item: ScrapeItem, post: AvailablePost) -> None:
+ async def post(self, scrape_item: ScrapeItem, /, post: AvailablePost) -> None:
title = self.create_separate_post_title(post.title, str(post.id), post.created_at)
scrape_item.setup_as_post(title)
for link in self.extract_links(post):
@@ -149,7 +148,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
return unique(iter_links())
def parse_url(
- self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True
+ self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool | None = None
) -> AbsoluteHttpURL:
return _clean_url(super().parse_url(link_str, relative_to, trim=trim))
diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.py
index 427926a6b..fbb0f992d 100644
--- a/cyberdrop_dl/crawlers/doodstream.py
+++ b/cyberdrop_dl/crawlers/doodstream.py
@@ -57,18 +57,13 @@ class DoodStreamCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- response, soup = await self.client._get_response_and_soup_cffi(self.DOMAIN, scrape_item.url)
+ async with self.request(scrape_item.url, impersonate=True) as resp:
+ actual_host = resp.url.host
+ soup = await resp.soup()
- host = self.parse_url(response.url).host
- assert host
- del response
-
- title: str = css.select_one_get_text(soup, "title")
- title = title.split("- DoodStream")[0].strip()
-
- file_id = get_file_id(soup)
- debrid_link = await self.get_download_url(host, soup)
+ title = css.page_title(soup, "DoodStream")
+ file_id = _get_file_id(soup)
+ debrid_link = await self.get_download_url(actual_host, soup)
filename, ext = self.get_filename_and_ext(f"{file_id}.mp4")
custom_filename = self.create_custom_filename(title, ext, file_id=file_id)
scrape_item.url = canonical_url
@@ -77,25 +72,23 @@ class DoodStreamCrawler(Crawler):
)
async def get_download_url(self, host: str, soup: BeautifulSoup) -> AbsoluteHttpURL:
- md5_path = get_md5_path(soup)
+ md5_path = _get_md5_path(soup)
api_url = API_MD5_ENTRYPOINT / md5_path
token = api_url.name
- async with self.request_limiter:
- new_soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, api_url.with_host(host))
- text = new_soup.get_text(strip=True)
+ text = await self.request_text(api_url.with_host(host), impersonate=True)
random_padding = "".join(random.choice(TOKEN_CHARS) for _ in range(10))
expire = int(datetime.now(UTC).timestamp() * 1000)
download_url = self.parse_url(text + random_padding)
return download_url.with_query(token=token, expiry=expire)
-def get_md5_path(soup: BeautifulSoup) -> str:
+def _get_md5_path(soup: BeautifulSoup) -> str:
js_text = css.select_one_get_text(soup, _SELECTORS.MD5_JS)
return get_text_between(js_text, "/pass_md5/", "'")
-def get_file_id(soup: BeautifulSoup) -> str:
+def _get_file_id(soup: BeautifulSoup) -> str:
js_text = css.select_one_get_text(soup, _SELECTORS.FILE_ID_JS)
_, file_id, _ = js_text.split("'file_id'")[-1].split("'", 2)
return file_id
diff --git a/cyberdrop_dl/crawlers/dropbox.py b/cyberdrop_dl/crawlers/dropbox.py
index 4740e161a..683c5bba0 100644
--- a/cyberdrop_dl/crawlers/dropbox.py
+++ b/cyberdrop_dl/crawlers/dropbox.py
@@ -46,7 +46,7 @@ class DropboxCrawler(Crawler):
preview_filename = scrape_item.url.query.get("preview")
match scrape_item.url.parts[1:]:
case ["s" | "sh", _, *_]:
- return await self.follow_redirect(scrape_item)
+ return await self.follow_redirect_w_head(scrape_item)
case ["scl", "fo", token1, token2]:
url = PRIMARY_URL / "scl/fo" / token1 / token2
if preview_filename:
@@ -78,32 +78,15 @@ class DropboxCrawler(Crawler):
raise ScrapeError(422, message="Folders download is not enabled")
scrape_item.url = item.view_url
- filename = item.filename or await self.get_content_disposition_name(item.download_url)
- if not filename:
- raise ScrapeError(422)
+ if item.filename:
+ filename = item.filename
+ else:
+ async with self.request(item.download_url) as resp:
+ filename = resp.filename
+
filename, ext = self.get_filename_and_ext(filename)
await self.handle_file(item.url, scrape_item, filename, ext, debrid_link=item.download_url)
- @error_handling_wrapper
- async def follow_redirect(self, scrape_item: ScrapeItem) -> None:
- scrape_item.url = await self.get_redict_url(scrape_item.url)
- await self.fetch(scrape_item)
-
- async def get_content_disposition_name(self, url: AbsoluteHttpURL) -> str | None:
- url = await self.get_redict_url(url)
- async with self.request_limiter:
- response = await self.client._get_head(self.DOMAIN, url)
- if response.content_disposition:
- return response.content_disposition.filename
-
- async def get_redict_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- async with self.request_limiter:
- headers = await self.client.get_head(self.DOMAIN, url)
- location = headers.get("location")
- if not location:
- raise ScrapeError(400)
- return self.parse_url(location)
-
@dataclass(frozen=True, slots=True)
class DropboxItem:
diff --git a/cyberdrop_dl/crawlers/e621.py b/cyberdrop_dl/crawlers/e621.py
index 1161f0791..b35b12f6d 100644
--- a/cyberdrop_dl/crawlers/e621.py
+++ b/cyberdrop_dl/crawlers/e621.py
@@ -3,8 +3,6 @@ from __future__ import annotations
import itertools
from typing import TYPE_CHECKING, Any, ClassVar
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl import __version__
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -28,10 +26,10 @@ class E621Crawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "e621.net"
FOLDER_DOMAIN: ClassVar[str] = "E621"
+ _RATE_LIMIT = 2, 1
def __post_init__(self) -> None:
self.headers = {"User-Agent": f"CyberDrop-DL/{__version__} (by B05FDD249DF29ED3)"}
- self.request_limiter = AsyncLimiter(2, 1)
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.query.get("tags"):
@@ -42,15 +40,13 @@ class E621Crawler(Crawler):
return await self.pool(scrape_item)
raise ValueError
- async def paginator(self, scrape_item: ScrapeItem) -> AsyncGenerator[list[dict[str, Any]]]:
+ async def _pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[list[dict[str, Any]]]:
"""Generator for album pages."""
initial_page = int(scrape_item.url.query.get("page", 1))
url = PRIMARY_URL / "posts.json"
for page in itertools.count(initial_page):
url = url.with_query(tags=scrape_item.url.query["tags"], page=page)
- async with self.request_limiter:
- json_resp: dict = await self.client.get_json(self.DOMAIN, url, self.headers)
-
+ json_resp: dict[str, Any] = await self.request_json(url, headers=self.headers)
posts: list[dict] = json_resp.get("posts", [])
if not posts:
break
@@ -63,7 +59,7 @@ class E621Crawler(Crawler):
title = self.create_title(tags.replace("+", " "))
scrape_item.setup_as_album(title)
- async for posts in self.paginator(scrape_item):
+ async for posts in self._pager(scrape_item):
for post in posts:
try:
file_url = post["file"]["url"]
@@ -81,9 +77,7 @@ class E621Crawler(Crawler):
"""Fetches posts from an e621 pool."""
pool_id = scrape_item.url.name
url = PRIMARY_URL / f"pools/{pool_id}.json"
- async with self.request_limiter:
- json_resp: dict = await self.client.get_json(self.DOMAIN, url, self.headers)
-
+ json_resp: dict[str, Any] = await self.request_json(url, headers=self.headers)
posts = json_resp.get("post_ids", [])
title: str = json_resp.get("name", "Unknown Pool").replace("_", " ")
scrape_item.setup_as_album(title)
@@ -91,7 +85,7 @@ class E621Crawler(Crawler):
for post_id in posts:
url = PRIMARY_URL / f"posts/{post_id}"
new_scrape_item = scrape_item.create_child(url)
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
@error_handling_wrapper
@@ -99,11 +93,9 @@ class E621Crawler(Crawler):
"""Fetches a single post by extracting the ID from the URL."""
post_id = scrape_item.url.name
url = PRIMARY_URL / f"posts/{post_id}.json"
- async with self.request_limiter:
- json_resp: dict = await self.client.get_json(self.DOMAIN, url, self.headers)
-
+ json_resp: dict[str, Any] = await self.request_json(url, headers=self.headers)
try:
- file_url = json_resp["post"]["file"]["url"]
+ file_url: str = json_resp["post"]["file"]["url"]
except KeyError:
raise ScrapeError(422) from None
diff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.py
index 11fedc80c..7a56d5f02 100644
--- a/cyberdrop_dl/crawlers/efukt.py
+++ b/cyberdrop_dl/crawlers/efukt.py
@@ -11,8 +11,6 @@ from cyberdrop_dl.utils.dates import to_timestamp
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
PRIMARY_URL = AbsoluteHttpURL("https://efukt.com")
@@ -60,15 +58,14 @@ class EfuktCrawler(Crawler):
scrape_item.setup_as_album(self.create_title(f"{title} [series]"))
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEO_THUMBS):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def video_image_or_gif(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
date_str = css.select_one_get_text(soup, _SELECTORS.DATE).split(" ", 1)[-1]
datetime = self._parse_date(date_str, "%m/%d/%y")
diff --git a/cyberdrop_dl/crawlers/ehentai.py b/cyberdrop_dl/crawlers/ehentai.py
index 4365b5fa2..97defd350 100644
--- a/cyberdrop_dl/crawlers/ehentai.py
+++ b/cyberdrop_dl/crawlers/ehentai.py
@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -53,27 +51,25 @@ class EHentaiCrawler(Crawler):
await self.set_no_warnings(scrape_item)
title: str = ""
+ gallery_id = scrape_item.url.parts[2]
scrape_item.url = scrape_item.url.with_query(None)
async for soup in self.web_pager(scrape_item.url):
if not title:
title = self.create_title(css.select_one_get_text(soup, _SELECTORS.TITLE))
date_str: str = css.select_one_get_text(soup, _SELECTORS.DATE)
- gallery_id = scrape_item.url.parts[2]
title = self.create_title(title, gallery_id)
scrape_item.setup_as_album(title, album_id=gallery_id)
scrape_item.possible_datetime = self.parse_date(date_str)
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUM_IMAGES):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
-
+ soup = await self.request_soup(scrape_item.url)
link_str: str = css.select_one_get_attr(soup, _SELECTORS.IMAGE, "src")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
@@ -84,6 +80,5 @@ class EHentaiCrawler(Crawler):
async def set_no_warnings(self, scrape_item: ScrapeItem) -> None:
"""Sets the no warnings cookie."""
url = scrape_item.url.update_query(nw="session")
- async with self.request_limiter:
- await self.client.get_soup(self.DOMAIN, url)
- self._warnings_set = True
+ async with self.request(url):
+ self._warnings_set = True
diff --git a/cyberdrop_dl/crawlers/eightmuses.py b/cyberdrop_dl/crawlers/eightmuses.py
index a00026e72..d7ae35939 100644
--- a/cyberdrop_dl/crawlers/eightmuses.py
+++ b/cyberdrop_dl/crawlers/eightmuses.py
@@ -33,8 +33,7 @@ class EightMusesCrawler(Crawler):
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
album_id = scrape_item.album_id
if not album_id:
diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.py
index a291712a8..8d9a322fa 100644
--- a/cyberdrop_dl/crawlers/eporner.py
+++ b/cyberdrop_dl/crawlers/eporner.py
@@ -116,7 +116,7 @@ class EpornerCrawler(Crawler):
url = canonical_url / part
async for soup in self.web_pager(url):
for _, new_scrape_item in self.iter_children(scrape_item, soup, selector, new_title_part=name):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def playlist(self, scrape_item: ScrapeItem, from_profile: bool = False) -> None:
@@ -131,7 +131,7 @@ class EpornerCrawler(Crawler):
scrape_item.setup_as_album(title)
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEO):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def gallery(self, scrape_item: ScrapeItem) -> None:
@@ -156,8 +156,7 @@ class EpornerCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
scrape_item.url = canonical_url
link_str = css.select_one_get_attr(soup, _SELECTORS.PHOTO, "href")
@@ -172,8 +171,7 @@ class EpornerCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
soup_str = soup.text
if "File has been removed due to copyright owner request" in soup_str:
diff --git a/cyberdrop_dl/crawlers/erome.py b/cyberdrop_dl/crawlers/erome.py
index e48dce10d..0829c694c 100644
--- a/cyberdrop_dl/crawlers/erome.py
+++ b/cyberdrop_dl/crawlers/erome.py
@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -48,15 +46,14 @@ class EromeCrawler(Crawler):
scrape_item.setup_as_profile(title)
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUM):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
album_id = scrape_item.url.parts[2]
results = await self.get_album_results(album_id)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
title_portion = css.select_one_get_text(soup, "title").rsplit(" - Porn")[0].strip()
if not title_portion:
diff --git a/cyberdrop_dl/crawlers/fapello.py b/cyberdrop_dl/crawlers/fapello.py
index 5c9b7bf4d..752fca3d4 100644
--- a/cyberdrop_dl/crawlers/fapello.py
+++ b/cyberdrop_dl/crawlers/fapello.py
@@ -2,16 +2,12 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -30,9 +26,7 @@ class FapelloCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
NEXT_PAGE_SELECTOR: ClassVar[str] = 'div[id="next_page"] a'
DOMAIN: ClassVar[str] = "fapello"
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(5, 1)
+ _RATE_LIMIT = 5, 1
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.name:
@@ -62,8 +56,7 @@ class FapelloCrawler(Crawler):
@error_handling_wrapper
async def post(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
content = css.select_one(soup, POST_CONTENT_SELECTOR)
for _, link in self.iter_tags(content, "img, source"):
diff --git a/cyberdrop_dl/crawlers/fileditch.py b/cyberdrop_dl/crawlers/fileditch.py
index 8d0181533..4d86f5dad 100644
--- a/cyberdrop_dl/crawlers/fileditch.py
+++ b/cyberdrop_dl/crawlers/fileditch.py
@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
DOWNLOAD_SELECTOR = "a[class*='download-button']"
@@ -32,8 +30,7 @@ class FileditchCrawler(Crawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_str: str = css.select_one_get_attr(soup, DOWNLOAD_SELECTOR, "href")
link = self.parse_url(link_str)
if link.path == HOMEPAGE_CATCHALL_FILE:
diff --git a/cyberdrop_dl/crawlers/files_vc.py b/cyberdrop_dl/crawlers/files_vc.py
index 571591a12..dae3b0d6c 100644
--- a/cyberdrop_dl/crawlers/files_vc.py
+++ b/cyberdrop_dl/crawlers/files_vc.py
@@ -2,8 +2,6 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, ClassVar
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils.utilities import error_handling_wrapper
@@ -22,9 +20,7 @@ class FilesVcCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "files.vc"
FOLDER_DOMAIN: ClassVar[str] = "FilesVC"
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(1, 1)
+ _RATE_LIMIT = 1, 1
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.path == "/d/dl" and scrape_item.url.query.get("hash"):
@@ -39,8 +35,7 @@ class FilesVcCrawler(Crawler):
hash = scrape_item.url.query["hash"]
api_url = API_ENTRYPOINT.joinpath("info").with_query(hash=hash)
- async with self.request_limiter:
- json_resp: dict[str, Any] = await self.client.get_json(self.DOMAIN, api_url)
+ json_resp: dict[str, Any] = await self.request_json(api_url)
filename, ext = self.get_filename_and_ext(json_resp["filename"], assume_ext=".zip")
scrape_item.possible_datetime = self.parse_date(json_resp["upload_time"])
diff --git a/cyberdrop_dl/crawlers/fourchan.py b/cyberdrop_dl/crawlers/fourchan.py
index 30049d066..5b18b723d 100644
--- a/cyberdrop_dl/crawlers/fourchan.py
+++ b/cyberdrop_dl/crawlers/fourchan.py
@@ -2,7 +2,6 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar, NotRequired, TypedDict, cast
-from aiolimiter import AsyncLimiter
from bs4 import BeautifulSoup
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
@@ -44,9 +43,7 @@ class FourChanCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "4chan"
_DOWNLOAD_SLOTS: ClassVar[int | None] = 1
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(3, 10)
+ _RATE_LIMIT = 3, 10
async def fetch(self, scrape_item: ScrapeItem) -> None:
if "thread" in scrape_item.url.parts:
@@ -59,8 +56,7 @@ class FourChanCrawler(Crawler):
async def thread(self, scrape_item: ScrapeItem) -> None:
board, _, thread_id = scrape_item.url.parts[1:4]
api_url = API_ENTRYPOINT / board / f"thread/{thread_id}.json"
- async with self.request_limiter:
- response: dict[str, list[Post]] = await self.client.get_json(self.DOMAIN, api_url, cache_disabled=True)
+ response: dict[str, list[Post]] = await self.request_json(api_url, cache_disabled=True)
if not response:
raise ScrapeError(404)
@@ -94,13 +90,11 @@ class FourChanCrawler(Crawler):
async def board(self, scrape_item: ScrapeItem) -> None:
board: str = scrape_item.url.parts[-1]
api_url = API_ENTRYPOINT / board / "threads.json"
- async with self.request_limiter:
- threads: list[ThreadList] = await self.client.get_json(self.DOMAIN, api_url, cache_disabled=True)
-
+ threads: list[ThreadList] = await self.request_json(api_url, cache_disabled=True)
scrape_item.setup_as_forum("")
for page in threads:
for thread in page["threads"]:
url = PRIMARY_URL / board / f"thread/{thread['no']}"
new_scrape_item = scrape_item.create_child(url)
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
diff --git a/cyberdrop_dl/crawlers/generic.py b/cyberdrop_dl/crawlers/generic.py
index 040b4382b..d696e05f6 100644
--- a/cyberdrop_dl/crawlers/generic.py
+++ b/cyberdrop_dl/crawlers/generic.py
@@ -63,9 +63,10 @@ class GenericCrawler(Crawler):
@error_handling_wrapper
@log_unsupported_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
- content_type = await self.get_content_type(scrape_item.url)
- if "html" in content_type:
- return await self.try_video_from_soup(scrape_item)
+ async with self.request(scrape_item.url) as resp:
+ content_type = resp.content_type
+ if "html" in content_type:
+ return await self.try_video_from_soup(scrape_item, await resp.soup())
filename, ext = guess_filename_and_ext(scrape_item.url, content_type)
if not ext:
@@ -75,18 +76,7 @@ class GenericCrawler(Crawler):
filename, _ = self.get_filename_and_ext(fullname.name)
await self.handle_file(scrape_item.url, scrape_item, filename, ext)
- async def get_content_type(self, url: AbsoluteHttpURL) -> str:
- async with self.request_limiter:
- headers = await self.client.get_head(self.DOMAIN, url)
- content_type: str = headers.get("Content-Type", "")
- if not content_type:
- raise ScrapeError(422)
- return content_type.lower()
-
- async def try_video_from_soup(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
-
+ async def try_video_from_soup(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> None:
try:
title = css.select_one_get_text(soup, "title").rsplit(" - ", 1)[0].rsplit("|", 1)[0]
link_str: str = css.select_one_get_attr(soup, VIDEO_SELECTOR, "src")
diff --git a/cyberdrop_dl/crawlers/girlsreleased.py b/cyberdrop_dl/crawlers/girlsreleased.py
index d23215fea..6246aa786 100644
--- a/cyberdrop_dl/crawlers/girlsreleased.py
+++ b/cyberdrop_dl/crawlers/girlsreleased.py
@@ -67,14 +67,10 @@ class GirlsReleasedCrawler(Crawler):
case _:
raise ValueError
- async def _api_request(self, api_url: AbsoluteHttpURL) -> dict[str, Any]:
- async with self.request_limiter:
- return await self.client.get_json(self.DOMAIN, api_url)
-
@error_handling_wrapper
async def set(self, scrape_item: ScrapeItem, set_id: str) -> None:
api_url = self.PRIMARY_URL / "api/0.1/set" / set_id
- set_data = (await self._api_request(api_url))["set"]
+ set_data = (await self.request_json(api_url))["set"]
self._handle_set(scrape_item, _parse_set(set_data))
@error_handling_wrapper
@@ -85,7 +81,7 @@ class GirlsReleasedCrawler(Crawler):
for page in itertools.count(0):
api_url = base_api_url / str(page)
- sets: list[dict[str, Any]] = (await self._api_request(api_url))["sets"]
+ sets: list[dict[str, Any]] = (await self.request_json(api_url))["sets"]
for set_data in sets:
set_ = _parse_set(set_data)
diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.py
index 9027cb0f6..9f96f13c8 100644
--- a/cyberdrop_dl/crawlers/gofile.py
+++ b/cyberdrop_dl/crawlers/gofile.py
@@ -7,8 +7,6 @@ from datetime import UTC, datetime, timedelta
from hashlib import sha256
from typing import TYPE_CHECKING, ClassVar, Literal, NotRequired, TypedDict, cast
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import FILE_HOST_ALBUM, AbsoluteHttpURL, ScrapeItem
from cyberdrop_dl.exceptions import DownloadError, PasswordProtectedError, ScrapeError
@@ -18,7 +16,7 @@ if TYPE_CHECKING:
from collections.abc import Mapping
-WT_REGEX = re.compile(r'appdata\.wt\s=\s"([^"]+)"')
+_FIND_WT = re.compile(r'appdata\.wt\s=\s"([^"]+)"').search
API_ENTRYPOINT = AbsoluteHttpURL("https://api.gofile.io")
GLOBAL_JS_URL = AbsoluteHttpURL("https://gofile.io/dist/js/global.js")
PRIMARY_URL = AbsoluteHttpURL("https://gofile.io")
@@ -87,13 +85,12 @@ class GoFileCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "gofile"
FOLDER_DOMAIN: ClassVar[str] = "GoFile"
- _RATE_LIMIT: ClassVar[tuple[float, float]] = 100, 60
+ _RATE_LIMIT: ClassVar[tuple[float, float]] = 4, 10
def __post_init__(self) -> None:
self.api_key = self.manager.config_manager.authentication_data.gofile.api_key
self.website_token = self.manager.cache_manager.get("gofile_website_token")
self.headers: dict[str, str] = {}
- self.request_limiter = AsyncLimiter(4, 6)
self._website_token_date = datetime.now(UTC) - timedelta(days=7)
async def async_startup(self) -> None:
@@ -117,8 +114,7 @@ class GoFileCrawler(Crawler):
api_url = api_url.update_query(password=sha256_password)
try:
- async with self.request_limiter:
- json_resp: ApiAlbumResponse = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)
+ json_resp: ApiAlbumResponse = await self.request_json(api_url, headers=self.headers)
except DownloadError as e:
if e.status != http.HTTPStatus.UNAUTHORIZED:
@@ -126,8 +122,7 @@ class GoFileCrawler(Crawler):
async with self.startup_lock:
await self.get_website_token(update=True)
api_url = api_url.update_query(wt=self.website_token)
- async with self.request_limiter:
- json_resp = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)
+ json_resp = await self.request_json(api_url, headers=self.headers)
album = get_album_data(json_resp)
if is_single_not_nested_file(scrape_item, album):
@@ -190,7 +185,7 @@ class GoFileCrawler(Crawler):
for url in subfolders:
subfolder = scrape_item.create_child(url)
- self.manager.task_group.create_task(self.run(subfolder))
+ self.create_task(self.run(subfolder))
tasks = []
for url in unavailable:
@@ -216,8 +211,7 @@ class GoFileCrawler(Crawler):
async def _get_new_api_key(self) -> str:
api_url = API_ENTRYPOINT / "accounts"
- async with self.request_limiter:
- json_resp = await self.client.post_data(self.DOMAIN, api_url, data={})
+ json_resp = await self.request_json(api_url, method="POST", data={})
if json_resp["status"] != "ok":
raise ScrapeError(401, "Couldn't generate GoFile API token", origin=api_url)
@@ -235,11 +229,11 @@ class GoFileCrawler(Crawler):
await self._update_website_token()
async def _update_website_token(self) -> None:
- async with self.request_limiter:
- text = await self.client.get_text(self.DOMAIN, GLOBAL_JS_URL)
- match = WT_REGEX.search(str(text))
+ text = await self.request_text(GLOBAL_JS_URL)
+ match = _FIND_WT(text)
if not match:
raise ScrapeError(401, "Couldn't generate GoFile websiteToken", origin=GLOBAL_JS_URL)
+
self.website_token = match.group(1)
self.manager.cache_manager.save("gofile_website_token", self.website_token)
self._website_token_date = datetime.now(UTC)
diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.py
index eeed3f962..e13c46c89 100644
--- a/cyberdrop_dl/crawlers/google_drive.py
+++ b/cyberdrop_dl/crawlers/google_drive.py
@@ -33,7 +33,7 @@
from __future__ import annotations
import asyncio
-from typing import TYPE_CHECKING, ClassVar, cast
+from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -121,8 +121,7 @@ class GoogleDriveCrawler(Crawler):
@error_handling_wrapper
async def folder(self, scrape_item: ScrapeItem, folder_id: str) -> None:
embeded_folder_url = (self.PRIMARY_URL / "embeddedfolderview").with_query(id=folder_id)
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, embeded_folder_url)
+ soup = await self.request_soup(embeded_folder_url)
folder_name = css.select_one_get_text(soup, "title")
title = self.create_title(folder_name, folder_id)
@@ -191,14 +190,10 @@ class GoogleDriveCrawler(Crawler):
# Use POST request to bypass "file is too large to scan. Would you still like to download this file" warning
method = "GET" if export_url.host == _DOCS_URL.host else "POST"
- # TODO: This request bypasses the config limiter. Use the new request method when PR #1251 is merged
- async with self.request_limiter, self.client._session.request(method, export_url) as resp:
- if not resp.ok or "html" in resp.content_type:
- await self.client.client_manager.check_http_status(resp)
+ async with self.request(export_url, method=method) as resp:
+ assert resp.ok and "html" not in resp.content_type
- direct_url = cast("AbsoluteHttpURL", resp.url)
- filename: str = resp.content_disposition.filename # type: ignore[reportAssignmentType,reportOptionalMemberAccess]
- return direct_url, filename
+ return resp.url, resp.filename
def _get_proper_doc_format(doc: str, format: str | None) -> str:
diff --git a/cyberdrop_dl/crawlers/hitomi_la.py b/cyberdrop_dl/crawlers/hitomi_la.py
index 985c3f221..20369edd3 100644
--- a/cyberdrop_dl/crawlers/hitomi_la.py
+++ b/cyberdrop_dl/crawlers/hitomi_la.py
@@ -8,8 +8,6 @@ from collections import defaultdict
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, ClassVar, NamedTuple, Required, TypedDict
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
@@ -91,7 +89,6 @@ class HitomiLaCrawler(Crawler):
def __post_init__(self) -> None:
self._semaphore = asyncio.Semaphore(3)
- self.request_limiter = AsyncLimiter(1, 1)
self.headers = {"Referer": str(PRIMARY_URL), "Origin": str(PRIMARY_URL)}
self._servers: Servers | None = None
@@ -117,10 +114,10 @@ class HitomiLaCrawler(Crawler):
nozomi_url = LTN_SERVER / colletion_type / f"{name}-{language}.nozomi"
scrape_item.setup_as_profile(title)
- async with self.request_limiter:
- response, _ = await self.client._get(self.DOMAIN, nozomi_url, self.headers)
+ async with self.request(nozomi_url, headers=self.headers) as response:
+ content = await response.read()
- for gallery_id in decode_nozomi_response(await response.read()):
+ for gallery_id in decode_nozomi_response(content):
new_scrape_item = scrape_item.create_child(PRIMARY_URL / f"galleries/{gallery_id}.html")
# await immediately to prevent overwhelming the downloader
# there could be thousands of galleries in the result
@@ -145,9 +142,9 @@ class HitomiLaCrawler(Crawler):
# This is partial implementation. Only parses tagged words, ex `female:dark_skin`
# Free form query searches are ignored
for nozomi_search_args in (parse_query_word(word) for word in search_query.split(" ") if ":" in word):
- async with self.request_limiter:
- response, _ = await self.client._get(self.DOMAIN, nozomi_search_args.url, self.headers)
- yield set(decode_nozomi_response(await response.read()))
+ async with self.request(nozomi_search_args.url, headers=self.headers) as response:
+ content = await response.read()
+ yield set(decode_nozomi_response(content))
@error_handling_wrapper
async def gallery(self, scrape_item: ScrapeItem) -> None:
@@ -165,8 +162,7 @@ class HitomiLaCrawler(Crawler):
async def get_gallery(self, gallery_id: str) -> Gallery:
gallery_url = LTN_SERVER / f"galleries/{gallery_id}.js"
- async with self.request_limiter:
- js_text = await self.client.get_text(self.DOMAIN, gallery_url, self.headers)
+ js_text = await self.request_text(gallery_url, headers=self.headers)
return json.loads(js_text.split("=", 1)[-1])
async def get_servers(self) -> Servers:
@@ -177,9 +173,8 @@ class HitomiLaCrawler(Crawler):
async def _get_servers(self) -> Servers:
# https://ltn.gold-usergeneratedcontent.net/gg.js
- async with self.request_limiter:
- js_text = await self.client.get_text(self.DOMAIN, LTN_SERVER / "gg.js", cache_disabled=True)
+ js_text = await self.request_text(LTN_SERVER / "gg.js", cache_disabled=True)
root, num, default_num = [
match_int_or_none(pattern, js_text) for pattern in (_REGEX.ROOT, _REGEX.NUM, _REGEX.DEFAULT_NUM)
]
diff --git a/cyberdrop_dl/crawlers/hotpic.py b/cyberdrop_dl/crawlers/hotpic.py
index 5591139c4..73b185241 100644
--- a/cyberdrop_dl/crawlers/hotpic.py
+++ b/cyberdrop_dl/crawlers/hotpic.py
@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -46,8 +44,7 @@ class HotPicCrawler(Crawler):
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
album_id = scrape_item.url.parts[2]
title = self.create_title(css.select_one_get_text(soup, "title").rsplit(" - ")[0], scrape_item.album_id)
@@ -61,8 +58,7 @@ class HotPicCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_str = css.select_one_get_attr(soup, _SELECTORS.IMAGE_OR_VIDEO, "src")
link = self.parse_url(link_str)
diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.py
index 1f55efcc6..f1d5b489c 100644
--- a/cyberdrop_dl/crawlers/imagebam.py
+++ b/cyberdrop_dl/crawlers/imagebam.py
@@ -49,7 +49,7 @@ class ImageBamCrawler(Crawler):
# this is to update the URL in the UI
# TODO: Add URL convertion methods as a pre_check step, before assigning them a task_id
# Several crawlers do something like this
- self.manager.task_group.create_task(self.run(scrape_item))
+ self.create_task(self.run(scrape_item))
return
match scrape_item.url.parts[1:]:
@@ -65,8 +65,7 @@ class ImageBamCrawler(Crawler):
@error_handling_wrapper
async def view(self, scrape_item: ScrapeItem) -> None:
# view URLs can be either a gallery or a single image.
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
if soup.select_one(_SELECTORS.IS_GALLERY):
return await self.gallery(scrape_item, soup)
await self.image(scrape_item, soup)
@@ -74,8 +73,7 @@ class ImageBamCrawler(Crawler):
@error_handling_wrapper
async def gallery(self, scrape_item: ScrapeItem, soup: BeautifulSoup | None = None) -> None:
if not soup:
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
gallery_name = css.select_one_get_text(soup, _SELECTORS.GALLERY_TITLE)
gallery_id = scrape_item.url.name
@@ -86,13 +84,12 @@ class ImageBamCrawler(Crawler):
def process_gallery_images(soup: BeautifulSoup) -> None:
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.THUMBNAILS):
if not self.check_album_results(new_scrape_item.url, results):
- self.manager.task_group.create_task(self.image(new_scrape_item))
+ self.create_task(self.image(new_scrape_item))
process_gallery_images(soup)
while next_page := css.select_one_get_attr_or_none(soup, _SELECTORS.NEXT_PAGE, "href"):
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, self.parse_url(next_page))
+ soup = await self.request_soup(self.parse_url(next_page))
process_gallery_images(soup)
@error_handling_wrapper
@@ -101,8 +98,7 @@ class ImageBamCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
image_tag = css.select_one(soup, _SELECTORS.IMAGE)
if not scrape_item.album_id and (gallery_info := soup.select_one(_SELECTORS.GALLERY_INFO)):
diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.py
index 47cf146e8..b28bfba0b 100644
--- a/cyberdrop_dl/crawlers/imgbb.py
+++ b/cyberdrop_dl/crawlers/imgbb.py
@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
PRIMARY_URL = AbsoluteHttpURL("https://ibb.co")
@@ -58,20 +56,19 @@ class ImgBBCrawler(Crawler):
first_page = self.parse_url(first_page_str)
for _, sub_album in self.iter_children(scrape_item, soup, ALBUM_PAGE_SELECTOR):
- self.manager.task_group.create_task(self.run(sub_album))
+ self.create_task(self.run(sub_album))
assert first_page
async for soup in self.web_pager(first_page):
for _, image in self.iter_children(scrape_item, soup, IMAGE_PAGE_SELECTOR):
- self.manager.task_group.create_task(self.run(image))
+ self.create_task(self.run(image))
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")
link = self.parse_url(link_str)
diff --git a/cyberdrop_dl/crawlers/imgbox.py b/cyberdrop_dl/crawlers/imgbox.py
index ddc72dff1..214acb82a 100644
--- a/cyberdrop_dl/crawlers/imgbox.py
+++ b/cyberdrop_dl/crawlers/imgbox.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+from typing import ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import FILE_HOST_ALBUM, AbsoluteHttpURL, ScrapeItem
@@ -8,9 +8,6 @@ from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
-if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
PRIMARY_URL = AbsoluteHttpURL("https://imgbox.com")
IMAGES_SELECTOR = "div#gallery-view-content img"
IMAGE_SELECTOR = "img[id=img]"
@@ -37,8 +34,7 @@ class ImgBoxCrawler(Crawler):
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
if "The specified gallery could not be found" in soup.text:
raise ScrapeError(404)
@@ -62,8 +58,7 @@ class ImgBoxCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")
link = self.parse_url(link_str)
diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.py
index 8fa3c388e..df16a5998 100644
--- a/cyberdrop_dl/crawlers/imgur.py
+++ b/cyberdrop_dl/crawlers/imgur.py
@@ -54,20 +54,17 @@ class ImgurCrawler(Crawler):
album_id = scrape_item.url.parts[-1]
title: str = ""
- async with self.request_limiter:
- await self.check_imgur_credits(scrape_item)
- api_url = API_ENTRYPOINT / "album" / album_id
- json_resp: dict[str, dict] = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)
+ await self.check_imgur_credits(scrape_item)
+ api_url = API_ENTRYPOINT / "album" / album_id
+ json_resp: dict[str, dict[str, Any]] = await self.request_json(api_url, headers=self.headers)
- title_part = json_resp["data"].get("title", album_id)
- title = self.create_title(title_part, album_id)
- scrape_item.setup_as_album(title, album_id=album_id)
+ title: str = json_resp["data"].get("title", album_id)
+ scrape_item.setup_as_album(self.create_title(title, album_id), album_id=album_id)
- async with self.request_limiter:
- api_url = API_ENTRYPOINT / "album" / album_id / "images"
- json_resp = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)
+ api_url = API_ENTRYPOINT / "album" / album_id / "images"
+ images: dict[str, list[dict[str, Any]]] = await self.request_json(api_url, headers=self.headers)
- for image in json_resp["data"]:
+ for image in images["data"]:
await self.process_image(scrape_item, image)
@error_handling_wrapper
@@ -77,10 +74,9 @@ class ImgurCrawler(Crawler):
raise LoginError(msg)
image_id = scrape_item.url.parts[-1]
- async with self.request_limiter:
- await self.check_imgur_credits(scrape_item)
- api_url = API_ENTRYPOINT / "image" / image_id
- json_resp = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)
+ await self.check_imgur_credits(scrape_item)
+ api_url = API_ENTRYPOINT / "image" / image_id
+ json_resp: dict[str, dict[str, Any]] = await self.request_json(api_url, headers=self.headers)
await self.process_image(scrape_item, json_resp["data"])
@@ -104,8 +100,7 @@ class ImgurCrawler(Crawler):
async def check_imgur_credits(self, _=None) -> None:
"""Checks the remaining credits."""
- credits_url = API_ENTRYPOINT / "credits"
- json_resp = await self.client.get_json(self.DOMAIN, credits_url, headers=self.headers)
+ json_resp = await self.request_json(API_ENTRYPOINT / "credits", headers=self.headers)
self.imgur_client_remaining = json_resp["data"]["ClientRemaining"]
if self.imgur_client_remaining < 100:
raise ScrapeError(429, "Imgur API rate limit reached")
diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.py
index 66d4fd7d5..2bf18a362 100644
--- a/cyberdrop_dl/crawlers/imx_to.py
+++ b/cyberdrop_dl/crawlers/imx_to.py
@@ -38,9 +38,7 @@ class ImxToCrawler(Crawler):
return
data = {"imgContinue": "Continue+to+image+...+"}
- async with self.request_limiter:
- soup = await self.client.post_data_get_soup(self.DOMAIN, scrape_item.url, data=data)
-
+ soup = await self.request_soup(scrape_item.url, method="POST", data=data)
link_str: str = css.select_one_get_attr(soup, IMG_SELECTOR, "src")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name, assume_ext=".jpg")
diff --git a/cyberdrop_dl/crawlers/incestflix.py b/cyberdrop_dl/crawlers/incestflix.py
index 7ebda098a..600b0886c 100644
--- a/cyberdrop_dl/crawlers/incestflix.py
+++ b/cyberdrop_dl/crawlers/incestflix.py
@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -44,8 +42,7 @@ class IncestflixCrawler(Crawler):
async def video(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
title: str = css.select_one_get_text(soup, _SELECTORS.TITLE)
link_str = css.select_one_get_attr(soup, _SELECTORS.VIDEO, "src")
url = self.parse_url(link_str)
@@ -61,4 +58,4 @@ class IncestflixCrawler(Crawler):
scrape_item.setup_as_album(title)
title_created = True
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEO_THUMBS):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
diff --git a/cyberdrop_dl/crawlers/influencer_bitches.py b/cyberdrop_dl/crawlers/influencer_bitches.py
index 3948d329d..fa8e48bb1 100644
--- a/cyberdrop_dl/crawlers/influencer_bitches.py
+++ b/cyberdrop_dl/crawlers/influencer_bitches.py
@@ -38,8 +38,7 @@ class InfluencerBitchesCrawler(Crawler):
@error_handling_wrapper
async def model(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
title_tag = soup.select_one(_SELECTORS.TITLE) or soup.select_one(_SELECTORS.ALTERNATIVE_TITLE)
assert title_tag
title: str = title_tag.get_text(strip=True).replace("leaks", "").strip()
diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.py
index 498b2d3a0..172799656 100644
--- a/cyberdrop_dl/crawlers/jpg5.py
+++ b/cyberdrop_dl/crawlers/jpg5.py
@@ -1,7 +1,5 @@
from __future__ import annotations
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from ._chevereto import CheveretoCrawler
@@ -28,8 +26,7 @@ class JPG5Crawler(CheveretoCrawler):
"jpg5.su",
)
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(1, 1)
+ _RATE_LIMIT = 1, 1
@classmethod
def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
diff --git a/cyberdrop_dl/crawlers/leakedzone.py b/cyberdrop_dl/crawlers/leakedzone.py
index be75fa98b..6dff01633 100644
--- a/cyberdrop_dl/crawlers/leakedzone.py
+++ b/cyberdrop_dl/crawlers/leakedzone.py
@@ -5,8 +5,6 @@ import dataclasses
import itertools
from typing import TYPE_CHECKING, Any, ClassVar
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.compat import IntEnum
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -64,6 +62,7 @@ class LeakedZoneCrawler(Crawler):
FOLDER_DOMAIN: ClassVar[str] = "LeakedZone"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
IMAGES_CDN: ClassVar[AbsoluteHttpURL] = IMAGES_CDN
+ _RATE_LIMIT = 3, 10
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
@@ -74,9 +73,6 @@ class LeakedZoneCrawler(Crawler):
case _:
raise ValueError
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(3, 10)
-
@classmethod
def get_encoded_video_url(cls, soup: BeautifulSoup) -> str:
js_text = css.select_one_get_text(soup, _SELECTORS.JW_PLAYER)
@@ -84,15 +80,15 @@ class LeakedZoneCrawler(Crawler):
@error_handling_wrapper
async def model(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
model_name: str = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME_FROM_PROFILE)
scrape_item.setup_as_profile(self.create_title(model_name))
headers = {"X-Requested-With": "XMLHttpRequest"}
for page in itertools.count(1):
- async with self.request_limiter:
- posts = await self.client.get_json(self.DOMAIN, scrape_item.url.with_query(page=page), headers=headers)
+ posts: list[dict[str, Any]] = await self.request_json(
+ scrape_item.url.with_query(page=page), headers=headers
+ )
# We may be able to omit the last request by just checking the number of posts
# Seens to always return 48 posts
if not posts:
@@ -111,8 +107,7 @@ class LeakedZoneCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item.url):
return
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
model_name = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME)
scrape_item.setup_as_album(self.create_title(model_name))
diff --git a/cyberdrop_dl/crawlers/luscious.py b/cyberdrop_dl/crawlers/luscious.py
index 78c097c26..e3014a485 100644
--- a/cyberdrop_dl/crawlers/luscious.py
+++ b/cyberdrop_dl/crawlers/luscious.py
@@ -39,7 +39,7 @@ class LusciousCrawler(Crawler):
return await self.search(scrape_item)
await self.album(scrape_item)
- async def create_graphql_query(self, operation: str, scrape_item: ScrapeItem, page: int = 1) -> str:
+ def create_graphql_query(self, operation: str, scrape_item: ScrapeItem, page: int = 1) -> str:
"""Creates a graphql query."""
assert operation in GRAPHQL_QUERIES
query = scrape_item.url.query
@@ -71,16 +71,14 @@ class LusciousCrawler(Crawler):
results = await self.get_album_results(album_id)
title: str = ""
query_name = "AlbumGet"
- async with self.request_limiter:
- query = await self.create_graphql_query(query_name, scrape_item)
- json_resp = await self.make_post_request(query_name, query)
-
+ query = self.create_graphql_query(query_name, scrape_item)
+ json_resp = await self._api_request(query_name, query)
title = self.create_title(json_resp["data"]["album"]["get"]["title"], album_id)
scrape_item.setup_as_album(title, album_id=album_id)
- async for json_resp in self.paginator(scrape_item, is_album=True):
- for item in json_resp["data"]["picture"]["list"]["items"]:
- link = self.parse_url(item["url_to_original"])
+ async for albums in self._pager(scrape_item, is_album=True):
+ for album in albums:
+ link = self.parse_url(album["url_to_original"])
if not self.check_album_results(link, results):
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
@@ -94,28 +92,31 @@ class LusciousCrawler(Crawler):
if not query:
raise ScrapeError(400, "No search query provided")
- async for json_data in self.paginator(scrape_item):
- for item in json_data["data"]["album"]["list"]["items"]:
- album_url = self.parse_url(item["url"])
+ async for results in self._pager(scrape_item):
+ for album in results:
+ album_url = self.parse_url(album["url"])
new_scrape_item = scrape_item.create_child(url=album_url)
await self.album(new_scrape_item)
- async def paginator(self, scrape_item: ScrapeItem, is_album: bool = False) -> AsyncGenerator[dict]:
+ async def _pager(self, scrape_item: ScrapeItem, is_album: bool = False) -> AsyncGenerator[list[dict[str, Any]]]:
"""Generator for album pages."""
initial_page = int(scrape_item.url.query.get("page", 1))
query_name = "PictureListInsideAlbum" if is_album else "AlbumListWithPeek"
data_name = "picture" if is_album else "album"
for page in itertools.count(initial_page):
- query: str = await self.create_graphql_query(query_name, scrape_item, page)
- json_resp = await self.make_post_request(query_name, query)
- yield json_resp
- if not json_resp["data"][data_name]["list"]["info"]["has_next_page"]:
+ query = self.create_graphql_query(query_name, scrape_item, page)
+ results: dict[str, Any] = (await self._api_request(query_name, query))["data"][data_name]["list"]
+ yield results["items"]
+ if not results["info"]["has_next_page"]:
break
- async def make_post_request(self, query_name: str, query: str) -> dict:
+ async def _api_request(self, query_name: str, query: str) -> dict[str, Any]:
api_url = GRAPHQL_URL.with_query(operationName=query_name)
- headers = {"Content-Type": "application/json"}
- async with self.request_limiter:
- json_resp = await self.client.post_data(self.DOMAIN, api_url, data=query, headers=headers)
+ json_resp = await self.request_json(
+ api_url,
+ method="POST",
+ data=query,
+ headers={"Content-Type": "application/json"},
+ )
log_debug(json_resp)
return json_resp
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 0391e8b30..215974037 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -3,7 +3,6 @@ from __future__ import annotations
import itertools
from typing import TYPE_CHECKING, Any, ClassVar
-from aiolimiter import AsyncLimiter
from mediafire import MediaFireApi, api
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
@@ -13,8 +12,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
DOWNLOAD_BUTTON_SELECTOR = "a[id=downloadButton]"
@@ -29,10 +26,10 @@ class MediaFireCrawler(Crawler):
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "mediafire"
+ _RATE_LIMIT = 5, 1
def __post_init__(self) -> None:
self.api = MediaFireApi()
- self.request_limiter = AsyncLimiter(5, 1)
async def fetch(self, scrape_item: ScrapeItem) -> None:
if "folder" in scrape_item.url.parts:
@@ -62,7 +59,7 @@ class MediaFireCrawler(Crawler):
date = self.parse_date(file["created"])
link = self.parse_url(file["links"]["normal_download"])
new_scrape_item = scrape_item.create_child(link, new_title_part=title, possible_datetime=date)
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
if not folder_contents["folder_content"]["more_chunks"] == "yes":
@@ -73,8 +70,7 @@ class MediaFireCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_tag = soup.select_one(DOWNLOAD_BUTTON_SELECTOR)
if not link_tag:
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index 9bba1210a..080ba05eb 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -7,7 +7,6 @@ It calls checks_complete_by_referer several times even if no request is going to
from __future__ import annotations
-import itertools
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, cast
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id
@@ -60,7 +59,7 @@ class MegaNzCrawler(Crawler):
return self.manager.auth_config.meganz.password or None
def _init_downloader(self) -> mega.MegaDownloader:
- self.downloader = dl = mega.MegaDownloader(self.manager, self.DOMAIN)
+ self.downloader = dl = mega.MegaDownloader(self.manager, self.DOMAIN) # type: ignore[reportIncompatibleVariableOverride]
dl.startup()
return dl
@@ -171,16 +170,15 @@ class MegaNzCrawler(Crawler):
) -> None:
folder_id, shared_key = scrape_item.url.name, scrape_item.url.fragment
- def exclude_node(fs_entry: tuple[Path, mega.Node]):
- node = fs_entry[1]
- if node["t"] != mega.NodeType.FILE:
- return True
- if single_file_id and node["h"] != single_file_id:
- return True
- return False
+ def filter_files():
+ for path, node in filesystem.items():
+ if node["t"] != mega.NodeType.FILE:
+ continue
+ if single_file_id and node["h"] != single_file_id:
+ continue
+ yield path, cast("mega.File", node)
- for path, node in itertools.filterfalse(exclude_node, filesystem.items()):
- file = cast("mega.File", node)
+ for path, file in filter_files():
file_id = file["h"]
file_fragment = f"{shared_key}/file/{file_id}"
canonical_url = scrape_item.url.with_fragment(file_fragment)
diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.py
index b0ca902f8..dfa380aa2 100644
--- a/cyberdrop_dl/crawlers/missav.py
+++ b/cyberdrop_dl/crawlers/missav.py
@@ -35,8 +35,7 @@ class MissAVCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url, impersonate=True)
title, date_str = open_graph.title(soup), open_graph.get("video_release_date", soup)
if dvd_code_tag := soup.select_one(DVD_CODE_SELECTOR):
diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.py
index c29aeb0e7..f8ea481d0 100644
--- a/cyberdrop_dl/crawlers/mixdrop.py
+++ b/cyberdrop_dl/crawlers/mixdrop.py
@@ -51,13 +51,11 @@ class MixDropCrawler(Crawler):
return
scrape_item.url = embed_url
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, video_url)
+ soup = await self.request_soup(video_url)
title = css.select_one_get_text(soup, _SELECTOR.FILE_NAME)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, embed_url)
+ soup = await self.request_soup(embed_url)
link = self.create_download_link(soup)
filename, ext = self.get_filename_and_ext(link.name)
diff --git a/cyberdrop_dl/crawlers/motherless.py b/cyberdrop_dl/crawlers/motherless.py
index 0861dcc5c..b8e592171 100644
--- a/cyberdrop_dl/crawlers/motherless.py
+++ b/cyberdrop_dl/crawlers/motherless.py
@@ -2,8 +2,6 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
@@ -43,9 +41,7 @@ class MotherlessCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
NEXT_PAGE_SELECTOR: ClassVar[str] = "div.pagination_link > a[rel=next]"
DOMAIN: ClassVar[str] = "motherless"
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(2, 1)
+ _RATE_LIMIT = 2, 1
async def fetch(self, scrape_item: ScrapeItem, collection_id: str = "") -> None:
parts = scrape_item.url.parts
@@ -87,13 +83,13 @@ class MotherlessCrawler(Crawler):
async for soup in self.web_pager(images_url):
check_soup(soup)
for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part="Images"):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
if is_homepage or "videos" in scrape_item.url.parts:
async for soup in self.web_pager(videos_url):
check_soup(soup)
for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part="Videos"):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def collection(self, scrape_item: ScrapeItem) -> None:
@@ -134,7 +130,7 @@ class MotherlessCrawler(Crawler):
scrape_item.setup_as_album(title, album_id=collection_id)
for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part=name):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def media(self, scrape_item: ScrapeItem) -> None:
@@ -144,8 +140,7 @@ class MotherlessCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
check_soup(soup)
media_info = self.process_media_soup(scrape_item, soup)
diff --git a/cyberdrop_dl/crawlers/nhentai.py b/cyberdrop_dl/crawlers/nhentai.py
index f5466e785..b08e070fc 100644
--- a/cyberdrop_dl/crawlers/nhentai.py
+++ b/cyberdrop_dl/crawlers/nhentai.py
@@ -3,8 +3,6 @@ from __future__ import annotations
import random
from typing import TYPE_CHECKING, Any, ClassVar
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import LoginError
@@ -41,9 +39,7 @@ class NHentaiCrawler(Crawler):
NEXT_PAGE_SELECTOR = "a.next"
DOMAIN = "nhentai.net"
FOLDER_DOMAIN = "nHentai"
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(4, 1)
+ _RATE_LIMIT = 4, 1
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
@@ -80,14 +76,11 @@ class NHentaiCrawler(Crawler):
async def gallery(self, scrape_item: ScrapeItem) -> None:
gallery_id = scrape_item.url.name
api_url = self.PRIMARY_URL / "api/gallery" / gallery_id
-
- async with self.request_limiter:
- json_resp: dict[str, Any] = await self.client.request_json_cffi(self.DOMAIN, api_url)
+ json_resp: dict[str, Any] = await self.request_json(api_url, impersonate=True)
titles: dict[str, str] = json_resp["title"]
title: str = titles.get("english") or titles.get("japanese") or titles["pretty"]
- title = self.create_title(title, gallery_id)
- scrape_item.setup_as_album(title, album_id=gallery_id)
+ scrape_item.setup_as_album(self.create_title(title, gallery_id), album_id=gallery_id)
scrape_item.possible_datetime = json_resp["upload_date"]
padding = max(3, len(str(json_resp["num_pages"])))
diff --git a/cyberdrop_dl/crawlers/noodle_magazine.py b/cyberdrop_dl/crawlers/noodle_magazine.py
index 94b67a29b..52960b611 100644
--- a/cyberdrop_dl/crawlers/noodle_magazine.py
+++ b/cyberdrop_dl/crawlers/noodle_magazine.py
@@ -4,9 +4,6 @@ import itertools
import json
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple
-from aiolimiter import AsyncLimiter
-from bs4 import BeautifulSoup
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
@@ -14,8 +11,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -44,9 +39,7 @@ class NoodleMagazineCrawler(Crawler):
DOMAIN: ClassVar[str] = "noodlemagazine"
FOLDER_DOMAIN: ClassVar[str] = "NoodleMagazine"
_DOWNLOAD_SLOTS: ClassVar[int | None] = 2
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(1, 3)
+ _RATE_LIMIT = 1, 3
async def fetch(self, scrape_item: ScrapeItem) -> None:
if "video" in scrape_item.url.parts:
@@ -63,8 +56,7 @@ class NoodleMagazineCrawler(Crawler):
for page in itertools.count(1, init_page):
n_videos = 0
page_url = scrape_item.url.with_query(p=page)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, page_url)
+ soup = await self.request_soup(page_url, impersonate=True)
if not title:
search_string: str = css.select_one_get_text(soup, SEARCH_STRING_SELECTOR)
@@ -76,7 +68,7 @@ class NoodleMagazineCrawler(Crawler):
if new_scrape_item.url not in seen_urls:
seen_urls.add(new_scrape_item.url)
n_videos += 1
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
if n_videos < 24:
break
@@ -85,8 +77,7 @@ class NoodleMagazineCrawler(Crawler):
async def video(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item.url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url, impersonate=True)
metadata_text = css.select_one(soup, METADATA_SELECTOR).get_text()
metadata = json.loads(metadata_text.strip())
diff --git a/cyberdrop_dl/crawlers/nudostartv.py b/cyberdrop_dl/crawlers/nudostartv.py
index 329f06786..b084f019f 100644
--- a/cyberdrop_dl/crawlers/nudostartv.py
+++ b/cyberdrop_dl/crawlers/nudostartv.py
@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
IMAGE_SELECTOR = "div[class=block-video] a img"
@@ -47,15 +45,14 @@ class NudoStarTVCrawler(Crawler):
raise ScrapeError(404)
for _, new_scrape_item in self.iter_children(scrape_item, soup, CONTENT_SELECTOR):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
diff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.py
index c6ee32b6a..fde93f958 100644
--- a/cyberdrop_dl/crawlers/odnoklassniki.py
+++ b/cyberdrop_dl/crawlers/odnoklassniki.py
@@ -71,8 +71,7 @@ class OdnoklassnikiCrawler(Crawler):
@error_handling_wrapper
async def channel(self, scrape_item: ScrapeItem, channel_str: str):
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url, _HEADERS)
+ soup = await self.request_soup(scrape_item.url, headers=_HEADERS)
channel_id = channel_str.removeprefix("c")
gwt_hash = get_text_between(css.select_one_get_text(soup, Selector.CHANNEL_HASH), 'gwtHash:"', '",')
@@ -107,26 +106,24 @@ class OdnoklassnikiCrawler(Crawler):
break
page += 1
- async with self.request_limiter:
- resp = await self.client._post_data(
- self.DOMAIN,
- page_url,
- _HEADERS,
- data={
- "fetch": "false",
- "st.page": page,
- "st.lastelem": last_element_id,
- "gwt.requested": gwt_hash,
- },
- )
- last_element_id = resp.headers.get("lastelem")
- content = await resp.text()
+ async with self.request(
+ page_url,
+ method="POST",
+ headers=_HEADERS,
+ data={
+ "fetch": "false",
+ "st.page": page,
+ "st.lastelem": last_element_id,
+ "gwt.requested": gwt_hash,
+ },
+ ) as resp:
+ last_element_id = resp.headers.get("lastelem")
+ content = await resp.text()
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str):
mobile_url = AbsoluteHttpURL(f"https://m.ok.ru/video/{video_id}")
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, mobile_url, _MOBILE_HEADERS)
+ soup = await self.request_soup(mobile_url, headers=_MOBILE_HEADERS)
_check_video_is_available(soup)
metadata: dict[str, Any] = json.loads(Selector.FLASHVARS(soup))["flashvars"]["metadata"]
diff --git a/cyberdrop_dl/crawlers/omegascans.py b/cyberdrop_dl/crawlers/omegascans.py
index 938b25fb5..3b48e7d4d 100644
--- a/cyberdrop_dl/crawlers/omegascans.py
+++ b/cyberdrop_dl/crawlers/omegascans.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import itertools
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -10,8 +10,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -43,8 +41,7 @@ class OmegaScansCrawler(Crawler):
@error_handling_wrapper
async def series(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
series_id = None
js_script = soup.select_one(JS_SELECTOR)
@@ -55,15 +52,14 @@ class OmegaScansCrawler(Crawler):
scrape_item.setup_as_album("", album_id=series_id)
# TODO: Add title
# title: str = ""
- for page in itertools.count(1):
- api_url = API_ENTRYPOINT.with_query(page=page, perPage=30, series_id=series_id)
- async with self.request_limiter:
- json_resp = await self.client.get_json(self.DOMAIN, api_url)
+ api_url = API_ENTRYPOINT.with_query(series_id=series_id, perPage=30)
+ for page in itertools.count(1):
+ json_resp: dict[str, Any] = await self.request_json(api_url.update_query(page=page))
for chapter in json_resp["data"]:
chapter_url = scrape_item.url / chapter["chapter_slug"]
new_scrape_item = scrape_item.create_child(chapter_url)
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
if json_resp["meta"]["current_page"] == json_resp["meta"]["last_page"]:
@@ -71,8 +67,7 @@ class OmegaScansCrawler(Crawler):
@error_handling_wrapper
async def chapter(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
if "This chapter is premium" in soup.get_text():
raise ScrapeError(401, "This chapter is premium")
diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.py
index 3c2a70730..da9be4401 100644
--- a/cyberdrop_dl/crawlers/onedrive.py
+++ b/cyberdrop_dl/crawlers/onedrive.py
@@ -4,7 +4,6 @@
from __future__ import annotations
-import json
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from functools import partial
@@ -132,13 +131,8 @@ class OneDriveCrawler(Crawler):
@error_handling_wrapper
async def share_link(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- headers = await self.client.get_head(self.DOMAIN, scrape_item.url)
- location = headers.get("location")
- if not location:
- raise ScrapeError(400)
og_share_link = scrape_item.url
- scrape_item.url = self.parse_url(location)
+ scrape_item.url = await self._get_redirect_url(scrape_item.url)
await self.link_with_credentials(scrape_item, og_share_link)
@error_handling_wrapper
@@ -200,7 +194,7 @@ class OneDriveCrawler(Crawler):
for access_details in subfolders:
api_url = create_api_url(access_details)
new_scrape_item = scrape_item.create_child(api_url)
- self.manager.task_group.create_task(self.process_access_details(new_scrape_item, access_details))
+ self.create_task(self.process_access_details(new_scrape_item, access_details))
scrape_item.add_children()
@error_handling_wrapper
@@ -213,20 +207,22 @@ class OneDriveCrawler(Crawler):
await self.handle_file(file.url, scrape_item, filename, ext, debrid_link=file.download_url)
async def make_api_request(self, api_url: AbsoluteHttpURL) -> dict[str, Any]:
- headers = {"Content-Type": "application/json"} | self.auth_headers
- async with self.request_limiter:
- json_resp: dict = await self.client.get_json(self.DOMAIN, api_url, headers=headers)
-
- return json_resp
+ return await self.request_json(
+ api_url,
+ headers={
+ "Content-Type": "application/json",
+ }
+ | self.auth_headers,
+ )
@error_handling_wrapper
async def get_badger_token(self, badger_url: AbsoluteHttpURL = BADGER_URL) -> None:
- new_headers = {"Content-Type": "application/json", "AppId": APP_ID}
- data = {"appId": APP_UUID}
- data_json = json.dumps(data)
- async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.DOMAIN, badger_url, headers=new_headers, data=data_json)
-
+ json_resp: dict[str, Any] = await self.request_json(
+ badger_url,
+ method="POST",
+ headers={"Content-Type": "application/json", "AppId": APP_ID},
+ json={"appId": APP_UUID},
+ )
badger_token: str = json_resp["token"]
badger_token_expires: str = json_resp["expiryTimeUtc"]
self.auth_headers = {"Prefer": "autoredeem", "Authorization": f"Badger {badger_token}"}
diff --git a/cyberdrop_dl/crawlers/pimpandhost.py b/cyberdrop_dl/crawlers/pimpandhost.py
index b74e2259e..156d880ec 100644
--- a/cyberdrop_dl/crawlers/pimpandhost.py
+++ b/cyberdrop_dl/crawlers/pimpandhost.py
@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
ALBUM_TITLE_SELECTOR = "span[class=author-header__album-name]"
@@ -47,12 +45,11 @@ class PimpAndHostCrawler(Crawler):
scrape_item.possible_datetime = self.parse_date(css.get_attr(date_tag, "title"), DATE_FORMAT)
for _, new_scrape_item in self.iter_children(scrape_item, soup, FILES_SELECTOR):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "data-src")
link = self.parse_url(link_str)
diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.py
index e23eb96e7..519cbb6ad 100644
--- a/cyberdrop_dl/crawlers/pixeldrain.py
+++ b/cyberdrop_dl/crawlers/pixeldrain.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import json
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -11,8 +11,6 @@ from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
API_ENTRYPOINT = AbsoluteHttpURL("https://pixeldrain.com/api/")
@@ -38,9 +36,8 @@ class PixelDrainCrawler(Crawler):
album_id = scrape_item.url.parts[2]
results = await self.get_album_results(album_id)
- async with self.request_limiter:
- api_url = API_ENTRYPOINT / "list" / scrape_item.url.parts[-1]
- json_resp = await self.client.get_json(self.DOMAIN, api_url)
+ api_url = API_ENTRYPOINT / "list" / scrape_item.url.parts[-1]
+ json_resp = await self.request_json(api_url)
log_debug(json_resp)
title = self.create_title(json_resp["title"], album_id)
@@ -70,10 +67,9 @@ class PixelDrainCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
+ api_url = API_ENTRYPOINT / "file" / scrape_item.url.name / "info"
try:
- async with self.request_limiter:
- api_url = API_ENTRYPOINT / "file" / scrape_item.url.name / "info"
- json_resp = await self.client.get_json(self.DOMAIN, api_url)
+ json_resp: dict[str, Any] = await self.request_json(api_url)
except DownloadError as e:
if e.status != 404:
raise
@@ -81,7 +77,7 @@ class PixelDrainCrawler(Crawler):
link = self.create_download_link(json_resp["id"])
scrape_item.possible_datetime = self.parse_date(json_resp["date_upload"])
- filename = json_resp["name"]
+ filename: str = json_resp["name"]
mime_type: str = json_resp["mime_type"]
try:
filename, ext = self.get_filename_and_ext(filename)
@@ -99,9 +95,8 @@ class PixelDrainCrawler(Crawler):
await self.handle_file(link, scrape_item, filename, ext)
async def text(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- api_url = API_ENTRYPOINT / "file" / scrape_item.url.parts[-1]
- text: str = await self.client.get_text(self.DOMAIN, api_url)
+ api_url = API_ENTRYPOINT / "file" / scrape_item.url.parts[-1]
+ text = await self.request_text(api_url)
for line in text.splitlines():
link = self.parse_url(line)
@@ -110,8 +105,7 @@ class PixelDrainCrawler(Crawler):
scrape_item.add_children()
async def filesystem(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
og_props = open_graph.parse(soup)
filename = og_props.title
diff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.py
index a6d9214fa..14cbe3642 100644
--- a/cyberdrop_dl/crawlers/pixhost.py
+++ b/cyberdrop_dl/crawlers/pixhost.py
@@ -41,15 +41,14 @@ class PixHostCrawler(Crawler):
match scrape_item.url.parts[1:]:
case ["gallery", gallery_id]:
return await self.gallery(scrape_item, gallery_id)
- case ["show", _]:
+ case ["show", _, *_]:
return await self.image(scrape_item)
case _:
raise ValueError
@error_handling_wrapper
async def gallery(self, scrape_item: ScrapeItem, album_id: str) -> None:
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
title = css.select_one_get_text(soup, _SELECTORS.GALLERY_TITLE)
title = self.create_title(title, album_id)
@@ -69,8 +68,7 @@ class PixHostCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_str = css.select_one_get_attr(soup, _SELECTORS.IMAGE, "src")
link = self.parse_url(link_str)
diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.py
index 2a98119fd..df7fa346a 100644
--- a/cyberdrop_dl/crawlers/pkmncards.py
+++ b/cyberdrop_dl/crawlers/pkmncards.py
@@ -7,8 +7,6 @@ from dataclasses import dataclass
from datetime import datetime
from typing import TYPE_CHECKING, ClassVar
-from bs4 import BeautifulSoup
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
@@ -17,7 +15,7 @@ from cyberdrop_dl.utils.dates import TimeStamp, to_timestamp
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup, Tag
+ from bs4 import Tag
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -146,13 +144,12 @@ class PkmncardsCrawler(Crawler):
download_url = self.parse_url(link_str)
simple_card = create_simple_card(title, download_url)
new_scrape_item = scrape_item.create_child(card_page_url)
- self.manager.task_group.create_task(self.handle_simple_card(new_scrape_item, simple_card))
+ self.create_task(self.handle_simple_card(new_scrape_item, simple_card))
scrape_item.add_children()
@error_handling_wrapper
async def card(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
name = css.select_one_get_text(soup, _SELECTORS.CARD_NAME)
number = css.select_one_get_text(soup, _SELECTORS.CARD_NUMBER)
@@ -177,8 +174,7 @@ class PkmncardsCrawler(Crawler):
async def handle_simple_card(self, scrape_item: ScrapeItem, simple_card: SimpleCard) -> None:
@error_handling_wrapper
async def get_card_set(self, scrape_item: ScrapeItem) -> CardSet:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
return create_set(soup)
async with self.set_locks[simple_card.set_abbr]:
diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.py
index 010a55e75..fba91091d 100644
--- a/cyberdrop_dl/crawlers/pmvhaven.py
+++ b/cyberdrop_dl/crawlers/pmvhaven.py
@@ -1,7 +1,8 @@
from __future__ import annotations
+import itertools
import json
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -74,17 +75,20 @@ class PMVHavenCrawler(Crawler):
# Playlist
# TODO: add pagination support for user playlists
- add_headers = {"Content-Type": "text/plain;charset=UTF-8"}
- add_data = json.dumps({"profile": username, "mode": "GetUser"})
- async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.DOMAIN, api_url, data=add_data, headers=add_headers)
+
+ json_resp: dict[str, Any] = await self.request_json(
+ api_url,
+ method="POST",
+ data=json.dumps({"profile": username, "mode": "GetUser"}),
+ headers={"Content-Type": "text/plain;charset=UTF-8"},
+ )
user_info: dict[str, dict] = json_resp["data"]
for playlist in user_info["playlists"]:
playlist_id = playlist["_id"]
link = PRIMARY_URL / "playlist" / playlist_id
new_scrape_item = scrape_item.create_child(link, new_title_part="Playlists")
- self.manager.task_group.create_task(self.playlist(new_scrape_item, add_suffix=False))
+ self.create_task(self.playlist(new_scrape_item, add_suffix=False))
scrape_item.add_children()
@error_handling_wrapper
@@ -151,11 +155,9 @@ class PMVHavenCrawler(Crawler):
video_id: str = scrape_item.url.name.rsplit("_", 1)[-1]
add_data = {"video": video_id, "mode": "InitVideo", "view": True}
api_url = API_ENTRYPOINT / "videoInput"
- async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.DOMAIN, api_url, data=add_data)
-
- videos = json_resp.get("video")
- video_info: dict = videos[0] if videos else {}
+ json_resp: dict[str, Any] = await self.request_json(api_url, method="POST", data=add_data)
+ videos: list[dict[str, Any]] = json_resp.get("video", [])
+ video_info = videos[0] if videos else {}
await self.process_video_info(scrape_item, video_info)
async def _generic_search_pager(self, scrape_item: ScrapeItem, add_data: dict, name: str, type: str = "") -> None:
@@ -205,18 +207,21 @@ class PMVHavenCrawler(Crawler):
is_profile = api_url.name == "profileInput"
add_headers = {"Content-Type": "text/plain;charset=UTF-8"} if is_profile else None
- while True:
+ for page in itertools.count(1):
data = {"index": page} | add_data
if is_profile:
data = json.dumps(data)
- async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.DOMAIN, api_url, data=data, headers=add_headers)
+ json_resp: dict[str, Any] = await self.request_json(
+ api_url,
+ method="POST",
+ data=data,
+ headers=add_headers,
+ )
has_videos = bool(json_resp[check_key])
if not has_videos:
break
yield json_resp
- page += 1
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.py
index 4d8916509..54184a6bc 100644
--- a/cyberdrop_dl/crawlers/pornhub.py
+++ b/cyberdrop_dl/crawlers/pornhub.py
@@ -177,7 +177,7 @@ class PornHubCrawler(Crawler):
async def iter_profile_pages(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL, selector: str) -> None:
async for soup in self.web_pager(url):
for _, new_scrape_item in self.iter_children(scrape_item, soup, selector):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:
@@ -243,7 +243,7 @@ class PornHubCrawler(Crawler):
title = self.create_title(title, playlist_id)
scrape_item.setup_as_album(f"{title} [playlist]", album_id=playlist_id)
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.PLAYLIST_VIDEOS, results=results):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
diff --git a/cyberdrop_dl/crawlers/pornpics.py b/cyberdrop_dl/crawlers/pornpics.py
index 2e334c7e1..50b1062e1 100644
--- a/cyberdrop_dl/crawlers/pornpics.py
+++ b/cyberdrop_dl/crawlers/pornpics.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-import json
+import itertools
from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
@@ -62,13 +62,13 @@ class PornPicsCrawler(Crawler):
title = self.create_title(f"{title} [{collection_type}]")
scrape_item.setup_as_profile(title)
- async for soup, items in self._web_pager(scrape_item):
+ async for soup, items in self._pager(scrape_item):
if soup:
update_scrape_item(soup)
for link in items:
new_scrape_item = scrape_item.create_child(link)
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
@error_handling_wrapper
@@ -77,8 +77,7 @@ class PornPicsCrawler(Crawler):
gallery_id = gallery_base.rsplit("-", 1)[-1]
results = await self.get_album_results(gallery_id)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
scrape_item.url = PRIMARY_URL / "galleries" / gallery_id # canonical URL
title = css.select_one_get_text(soup, "h1")
@@ -96,35 +95,31 @@ class PornPicsCrawler(Crawler):
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~``
- async def _web_pager(
+ async def _pager(
self, scrape_item: ScrapeItem
) -> AsyncGenerator[tuple[BeautifulSoup | None, tuple[AbsoluteHttpURL, ...]]]:
"""Generator of website pages."""
limit = 20
- page_url = scrape_item.url.without_query_params("limit", "offset")
- offset = int(scrape_item.url.query.get("offset") or 0)
+ page_url = scrape_item.url.without_query_params("limit", "offset").with_query(limit=limit)
+ init_offset = int(scrape_item.url.query.get("offset") or 0)
- async def get_items(current_page: AbsoluteHttpURL) -> tuple[BeautifulSoup | None, tuple[AbsoluteHttpURL, ...]]:
- offset = current_page.query.get("offset")
+ async def get_items(offset: int) -> tuple[BeautifulSoup | None, tuple[AbsoluteHttpURL, ...]]:
+ current_page = page_url.update_query(offset=offset)
if not offset: # offset == 0 does not return JSON
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, current_page)
+ soup = await self.request_soup(current_page)
items = soup.select(IMAGE_SELECTOR)
return soup, tuple(self.parse_url(css.get_attr(image, "href")) for image in items)
- async with self.request_limiter:
- # The response is JSON but the "content-type" is wrong so we have to request it as text
- json_resp = await self.client.get_text(self.DOMAIN, current_page)
- json_resp = json.loads(json_resp)
+ # The response is JSON but the "content-type" is wrong
+ async with self.request(current_page) as resp:
+ json_resp = await resp.json(content_type=False)
return None, tuple(self.parse_url(g["g_url"]) for g in json_resp)
- while True:
- soup, items = await get_items(page_url)
+ for offset in itertools.count(init_offset, limit):
+ soup, items = await get_items(offset)
yield soup, items
if len(items) < limit:
break
- offset += limit
- page_url = page_url.update_query(offset=offset, limit=limit)
def is_cdn(url: AbsoluteHttpURL) -> bool:
diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.py
index c7c9b2fa7..172a671cb 100644
--- a/cyberdrop_dl/crawlers/porntrex.py
+++ b/cyberdrop_dl/crawlers/porntrex.py
@@ -4,8 +4,6 @@ import itertools
import re
from typing import TYPE_CHECKING, ClassVar, NamedTuple
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
@@ -63,9 +61,7 @@ class PorntrexCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
NEXT_PAGE_SELECTOR: ClassVar[str] = _SELECTORS.NEXT_PAGE
DOMAIN: ClassVar[str] = "porntrex"
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(3, 10)
+ _RATE_LIMIT = 3, 10
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.name: # The ending slash is necessary or we get a 404 error
@@ -84,8 +80,7 @@ class PorntrexCrawler(Crawler):
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
album_id = scrape_item.url.parts[2]
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
if "This album is a private album" in soup.text:
raise ScrapeError(401, "Private album")
@@ -108,8 +103,7 @@ class PorntrexCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
video = get_video_info(soup)
link = self.parse_url(video.url)
@@ -122,8 +116,7 @@ class PorntrexCrawler(Crawler):
@error_handling_wrapper
async def collection(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
if "models" in scrape_item.url.parts:
title: str = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME).title()
@@ -151,7 +144,7 @@ class PorntrexCrawler(Crawler):
last_page: int = int(last_page_tag[-1].get_text(strip=True)) if last_page_tag else 1
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEOS_OR_ALBUMS):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
await self.proccess_additional_pages(scrape_item, last_page)
@@ -161,11 +154,10 @@ class PorntrexCrawler(Crawler):
elif "members" in scrape_item.url.parts:
albums_url = scrape_item.url / "albums"
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, albums_url)
+ soup = await self.request_soup(albums_url)
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUMS, new_title_part="albums"):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
async def proccess_additional_pages(self, scrape_item: ScrapeItem, last_page: int, **kwargs: str) -> None:
block_id: str = "list_videos_common_videos_list_norm"
@@ -200,11 +192,10 @@ class PorntrexCrawler(Crawler):
if page > last_page:
break
page_url = page_url.update_query({from_param_name: page})
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, page_url)
+ soup = await self.request_soup(page_url)
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEOS_OR_ALBUMS):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
def get_video_info(soup: BeautifulSoup) -> Video:
diff --git a/cyberdrop_dl/crawlers/postimg.py b/cyberdrop_dl/crawlers/postimg.py
index 6a2a61b64..bf1280470 100644
--- a/cyberdrop_dl/crawlers/postimg.py
+++ b/cyberdrop_dl/crawlers/postimg.py
@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
PRIMARY_URL = AbsoluteHttpURL("https://postimages.org/")
@@ -39,13 +37,15 @@ class PostImgCrawler(Crawler):
async def album(self, scrape_item: ScrapeItem) -> None:
data = {"action": "list", "album": scrape_item.url.raw_name, "page": 0}
title: str = ""
+ album_id = scrape_item.url.parts[2]
for page in itertools.count(1):
- data["page"] = page
- async with self.request_limiter:
- json_resp = await self.client.post_data(self.DOMAIN, API_ENTRYPOINT, data=data)
+ json_resp = await self.request_json(
+ API_ENTRYPOINT,
+ method="POST",
+ data=data | {"page": page},
+ )
if not title:
- album_id = scrape_item.url.parts[2]
title = self.create_title(scrape_item.url.name, album_id)
scrape_item.setup_as_album(title, album_id=album_id)
@@ -64,8 +64,7 @@ class PostImgCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_str: str = css.select_one_get_attr(soup, DOWNLOAD_BUTTON_SELECTOR, "href")
link = self.parse_url(link_str).with_query(None)
diff --git a/cyberdrop_dl/crawlers/realbooru.py b/cyberdrop_dl/crawlers/realbooru.py
index 7791481d5..a9d94458a 100644
--- a/cyberdrop_dl/crawlers/realbooru.py
+++ b/cyberdrop_dl/crawlers/realbooru.py
@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -51,12 +49,11 @@ class RealBooruCrawler(Crawler):
async for soup in self.web_pager(scrape_item.url, relative_to=scrape_item.url):
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.CONTENT):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_str = css.select_one_get_attr(soup, _SELECTORS.IMAGE_OR_VIDEO, "src")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
diff --git a/cyberdrop_dl/crawlers/realdebrid.py b/cyberdrop_dl/crawlers/realdebrid.py
index b514e1f26..305e6e42a 100644
--- a/cyberdrop_dl/crawlers/realdebrid.py
+++ b/cyberdrop_dl/crawlers/realdebrid.py
@@ -14,9 +14,6 @@ import re
from re import Pattern
from typing import TYPE_CHECKING, Any, ClassVar
-import aiohttp
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import RealDebridError
@@ -24,6 +21,7 @@ from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
+ from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
PRIMARY_URL = AbsoluteHttpURL("https://real-debrid.com")
@@ -74,12 +72,12 @@ class RealDebridCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "real-debrid"
FOLDER_DOMAIN: ClassVar[str] = "RealDebrid"
+ _RATE_LIMIT = 250, 60
def __post_init__(self) -> None:
self._api_token = token = self.manager.auth_config.realdebrid.api_key
self._supported_folder_url_regex: Pattern
self._supported_url_regex: Pattern
- self.request_limiter = AsyncLimiter(250, 60)
self.disabled = not bool(token)
self._headers = {"Authorization": f"Bearer {token}", "User-Agent": "CyberDrop-DL"}
@@ -103,11 +101,13 @@ class RealDebridCrawler(Crawler):
if self.disabled:
return
try:
- files_r, folders_r = await asyncio.gather(
- self._api_request("hosts/regex"), self._api_request("hosts/regexFolder")
+ responses: tuple[list[str], list[str]] = await asyncio.gather(
+ self._api_request("hosts/regex"),
+ self._api_request("hosts/regexFolder"),
)
- file_regex: list[str] = [pattern[1:-1] for pattern in files_r]
- folder_regex: list[str] = [pattern[1:-1] for pattern in folders_r]
+
+ file_regex = [pattern[1:-1] for pattern in responses[0]]
+ folder_regex = [pattern[1:-1] for pattern in responses[1]]
self._supported_url_regex = re.compile("|".join(file_regex + folder_regex))
self._supported_folder_url_regex = re.compile("|".join(folder_regex))
except Exception as e:
@@ -159,12 +159,27 @@ class RealDebridCrawler(Crawler):
async def _api_request(self, path: str, /, **data: Any) -> Any:
method = "POST" if data else "GET"
- data_ = data or None
- async with self.request_limiter, self.client._session.disabled():
- response = await self.client._session.request(
- method, _API_ENTRYPOINT / path, headers=self._headers, data=data_
- )
- return await _handle_api_response(response)
+
+ async with self.request(
+ _API_ENTRYPOINT / path,
+ method=method,
+ headers=self._headers,
+ data=data or None,
+ cache_disabled=True,
+ ) as resp:
+ return await self._handle_api_response(resp)
+
+ async def _handle_api_response(self, response: AbstractResponse) -> Any:
+ if "json" in response.content_type:
+ json_resp: dict[str, Any] = await response.json()
+ if code := json_resp.get("error_code"):
+ code = 7 if code == 16 else code
+ msg = _ERROR_CODES.get(code, "Unknown error")
+ raise RealDebridError(response.url, code, msg) from None
+ else:
+ return json_resp
+
+ await self.client.client_manager.check_http_status(response)
def _guess_folder(url: AbsoluteHttpURL) -> str:
@@ -235,22 +250,3 @@ def _flatten_url(original_url: AbsoluteHttpURL, host: str) -> AbsoluteHttpURL:
flatten_url = flatten_url / "frag" / original_url.fragment
return flatten_url
-
-
-async def _handle_api_response(response: aiohttp.ClientResponse) -> Any:
- try:
- json_resp: dict[str, Any] = await response.json()
- try:
- response.raise_for_status()
- except aiohttp.ClientResponseError:
- code = json_resp.get("error_code")
- if not code:
- raise
- code = 7 if code == 16 else code
- msg = _ERROR_CODES.get(code, "Unknown error")
- raise RealDebridError(response.url, code, msg) from None
- else:
- return json_resp
- except AttributeError:
- response.raise_for_status()
- return await response.text()
diff --git a/cyberdrop_dl/crawlers/reddit.py b/cyberdrop_dl/crawlers/reddit.py
index 6e0ea74c5..d78824ab5 100644
--- a/cyberdrop_dl/crawlers/reddit.py
+++ b/cyberdrop_dl/crawlers/reddit.py
@@ -4,7 +4,6 @@ from contextlib import contextmanager
from typing import TYPE_CHECKING, ClassVar, NotRequired, TypedDict
import asyncprawcore
-from aiolimiter import AsyncLimiter
from asyncpraw import Reddit
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
@@ -52,11 +51,11 @@ class RedditCrawler(Crawler):
DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{title}"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "reddit"
+ _RATE_LIMIT = 5, 1
def __post_init__(self) -> None:
self.reddit_personal_use_script = self.manager.config_manager.authentication_data.reddit.personal_use_script
self.reddit_secret = self.manager.config_manager.authentication_data.reddit.secret
- self.request_limiter = AsyncLimiter(5, 1)
self.logged_in = False
async def async_startup(self) -> None:
@@ -167,7 +166,10 @@ class RedditCrawler(Crawler):
try:
filename, ext = self.get_filename_and_ext(url.name)
except NoExtensionError:
- scrape_item.url = url = await self.get_final_location(url)
+ async with self.request(url) as resp:
+ if url == resp.url:
+ raise
+ scrape_item.url = url = resp.url
with asyncpraw_error_handle():
post = await self._reddit.submission(url=str(url))
@@ -175,17 +177,6 @@ class RedditCrawler(Crawler):
await self.handle_file(url, scrape_item, filename, ext)
- async def get_final_location(self, url) -> AbsoluteHttpURL:
- headers = await self.client.get_head(self.DOMAIN, url)
- content_type = headers.get("Content-Type", "")
- if any(s in content_type.lower() for s in ("html", "text")):
- response, _ = await self.client._get_response_and_soup(self.DOMAIN, url)
- return AbsoluteHttpURL(response.url)
- location = headers.get("location")
- if not location:
- raise ScrapeError(422)
- return self.parse_url(location)
-
@error_handling_wrapper
async def check_login(self, _) -> None:
if not (self.reddit_personal_use_script and self.reddit_secret):
diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.py
index 06ef47d28..f660b18d6 100644
--- a/cyberdrop_dl/crawlers/redgifs.py
+++ b/cyberdrop_dl/crawlers/redgifs.py
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
# Primary URL needs `www.` to prevent redirect
PRIMARY_URL = AbsoluteHttpURL("https://www.redgifs.com/")
-API_ENTRYPOINT = AbsoluteHttpURL("https://api.redgifs.com/")
+API_ENTRYPOINT = AbsoluteHttpURL("https://api.redgifs.com/v2")
class Links(TypedDict, total=False):
@@ -53,7 +53,7 @@ class RedGifsCrawler(Crawler):
self.headers = {}
async def async_startup(self) -> None:
- await self.get_auth_token(API_ENTRYPOINT / "v2/auth/temporary")
+ await self.get_auth_token(API_ENTRYPOINT / "auth/temporary")
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
@@ -64,7 +64,7 @@ class RedGifsCrawler(Crawler):
if self.is_self_subdomain(scrape_item.url) and len(scrape_item.url.parts) == 2:
scrape_item.url = _canonical_url(scrape_item.url.name)
- self.manager.task_group.create_task(self.run(scrape_item))
+ self.create_task(self.run(scrape_item))
return
raise ValueError
@@ -83,11 +83,8 @@ class RedGifsCrawler(Crawler):
async def _user_profile_pager(self, user_id: str) -> AsyncIterable[list[Gif]]:
total_pages: int | None = None
for page in itertools.count(1):
- async with self.request_limiter:
- api_url = API_ENTRYPOINT / "v2/users" / user_id / "search"
- api_url = api_url.with_query(order="new", page=page)
- json_resp: dict[str, Any] = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)
-
+ api_url = (API_ENTRYPOINT / "users" / user_id / "search").with_query(order="new", page=page)
+ json_resp: dict[str, Any] = await self.request_json(api_url, headers=self.headers)
gifs_in_current_page = [Gif.from_dict(gif) for gif in json_resp["gifs"]]
yield gifs_in_current_page
@@ -107,10 +104,8 @@ class RedGifsCrawler(Crawler):
return
scrape_item.url = canonical_url
- async with self.request_limiter:
- api_url = API_ENTRYPOINT / "v2/gifs" / post_id
- json_resp: dict[str, dict] = await self.client.get_json(self.DOMAIN, api_url, headers=self.headers)
-
+ api_url = API_ENTRYPOINT / "gifs" / post_id
+ json_resp: dict[str, dict] = await self.request_json(api_url, headers=self.headers)
gif = Gif.from_dict(json_resp["gif"])
if gif.title:
scrape_item.setup_as_album(self.create_title(gif.title))
@@ -123,8 +118,7 @@ class RedGifsCrawler(Crawler):
@error_handling_wrapper
async def get_auth_token(self, token_url: AbsoluteHttpURL) -> None:
- async with self.request_limiter:
- json_obj = await self.client.get_json(self.DOMAIN, token_url)
+ json_obj: dict[str, Any] = await self.request_json(token_url)
token: str = json_obj["token"]
self.headers = {"Authorization": f"Bearer {token}"}
diff --git a/cyberdrop_dl/crawlers/rule34vault.py b/cyberdrop_dl/crawlers/rule34vault.py
index d442dd8e6..fb9f69015 100644
--- a/cyberdrop_dl/crawlers/rule34vault.py
+++ b/cyberdrop_dl/crawlers/rule34vault.py
@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -52,8 +50,7 @@ class Rule34VaultCrawler(Crawler):
for page in itertools.count(init_page):
url = scrape_item.url.with_query(page=page)
n_images = 0
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, url)
+ soup = await self.request_soup(url)
if not title:
if is_playlist:
@@ -67,7 +64,7 @@ class Rule34VaultCrawler(Crawler):
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.CONTENT):
n_images += 1
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
if n_images < 30:
break
@@ -78,8 +75,7 @@ class Rule34VaultCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
if date_tag := soup.select_one(_SELECTORS.DATE):
scrape_item.possible_datetime = self.parse_date(date_tag.text, "%b %d, %Y, %I:%M:%S %p")
diff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.py
index c37ea77f4..13a331373 100644
--- a/cyberdrop_dl/crawlers/rule34video.py
+++ b/cyberdrop_dl/crawlers/rule34video.py
@@ -84,7 +84,7 @@ class Rule34VideoCrawler(Crawler):
scrape_item.setup_as_album(title)
for _, new_scrape_item in self.iter_children(scrape_item, soup, PLAYLIST_ITEM_SELECTOR):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
@@ -94,8 +94,7 @@ class Rule34VideoCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
scrape_item.url = canonical_url
video = _parse_video(soup)
diff --git a/cyberdrop_dl/crawlers/rule34xxx.py b/cyberdrop_dl/crawlers/rule34xxx.py
index c538c2b4a..800c34966 100644
--- a/cyberdrop_dl/crawlers/rule34xxx.py
+++ b/cyberdrop_dl/crawlers/rule34xxx.py
@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
PRIMARY_URL = AbsoluteHttpURL("https://rule34.xxx")
@@ -53,12 +51,11 @@ class Rule34XXXCrawler(Crawler):
scrape_item.setup_as_album(title)
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.CONTENT):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
date_str = css.select_one_get_text(soup, _SELECTORS.DATE).removeprefix("Posted: ")
scrape_item.possible_datetime = self.parse_date(date_str)
diff --git a/cyberdrop_dl/crawlers/saint.py b/cyberdrop_dl/crawlers/saint.py
index 938f039c4..408e11bd7 100644
--- a/cyberdrop_dl/crawlers/saint.py
+++ b/cyberdrop_dl/crawlers/saint.py
@@ -51,8 +51,7 @@ class SaintCrawler(Crawler):
async def album(self, scrape_item: ScrapeItem) -> None:
album_id = scrape_item.url.parts[2]
results = await self.get_album_results(album_id)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
title_portion = css.select_one_get_text(soup, "title").rsplit(" - Saint Video Hosting")[0].strip()
if not title_portion:
@@ -66,7 +65,7 @@ class SaintCrawler(Crawler):
link = self.parse_url(link_str)
if not self.check_album_results(link, results):
new_scrape_item = scrape_item.create_child(link)
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
@error_handling_wrapper
@@ -74,8 +73,7 @@ class SaintCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
try:
link_str: str = css.select_one_get_attr(soup, _SELECTORS.EMBED_SRC, "src")
except AssertionError:
@@ -88,8 +86,7 @@ class SaintCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
try:
link_str: str = css.select_one_get_attr(soup, _SELECTORS.DOWNLOAD_BUTTON, "href")
except (AttributeError, css.SelectorError):
diff --git a/cyberdrop_dl/crawlers/scrolller.py b/cyberdrop_dl/crawlers/scrolller.py
index 484f8cc60..8b35239cf 100644
--- a/cyberdrop_dl/crawlers/scrolller.py
+++ b/cyberdrop_dl/crawlers/scrolller.py
@@ -72,10 +72,14 @@ class ScrolllerCrawler(Crawler):
while True:
request_body["variables"]["iterator"] = iterator
- data: dict[str, dict] = await self.client.post_data(
- self.DOMAIN, API_ENTRYPOINT, data=json.dumps(request_body)
- )
- items: list[dict] = data["data"]["getSubreddit"]["children"]["items"] if data else []
+ data: dict[str, dict] = (
+ await self.request_json(
+ API_ENTRYPOINT,
+ method="POST",
+ data=json.dumps(request_body),
+ )
+ )["data"]
+ items: list[dict] = data["getSubreddit"]["children"]["items"] if data else []
if not items:
break
@@ -94,7 +98,7 @@ class ScrolllerCrawler(Crawler):
scrape_item.add_children()
prev_iterator = iterator
- iterator = data["data"]["getSubreddit"]["children"]["iterator"]
+ iterator = data["getSubreddit"]["children"]["iterator"]
if iterator is None or iterator == prev_iterator or iterations > 0:
break
diff --git a/cyberdrop_dl/crawlers/send_now.py b/cyberdrop_dl/crawlers/send_now.py
index dd5542ce4..a8ac9b288 100644
--- a/cyberdrop_dl/crawlers/send_now.py
+++ b/cyberdrop_dl/crawlers/send_now.py
@@ -7,8 +7,6 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from curl_cffi.requests.models import Response as CurlResponse
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
PRIMARY_URL = AbsoluteHttpURL("https://send.now/")
@@ -29,22 +27,35 @@ class SendNowCrawler(Crawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
file_id = scrape_item.url.name
- data = {"op": "download2", "id": file_id, "rand": "", "referer": "", "method_free": "", "method_premium": ""}
- params = {"allow_redirects": False, "stream": True}
-
- async with self.request_limiter:
- response: CurlResponse = await self.client.post_data_cffi(
- self.DOMAIN, scrape_item.url, data=data, request_params=params
- )
- debrid_link = self.parse_url(response.headers.get("location"))
+ async with self.request(
+ scrape_item.url,
+ method="POST",
+ impersonate=True,
+ data={
+ "op": "download2",
+ "id": file_id,
+ "rand": "",
+ "referer": "",
+ "method_free": "",
+ "method_premium": "",
+ },
+ request_params={"allow_redirects": False, "stream": True},
+ ) as resp:
+ debrid_link = resp.location
+
+ assert debrid_link
filename, ext = self.get_filename_and_ext(debrid_link.name, assume_ext=".zip")
await self.handle_file(scrape_item.url, scrape_item, filename, ext, debrid_link=debrid_link)
- async def get_cookies(self, scrape_item: ScrapeItem) -> None:
+ async def _get_cookies(self, scrape_item: ScrapeItem) -> None:
+ if self.got_cookies:
+ return
async with self.startup_lock:
- if not self.got_cookies:
- async with self.request_limiter:
- await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)
- cookies = self.manager.client_manager.cookies.filter_cookies(PRIMARY_URL)
- self.got_cookies = bool(cookies)
+ if self.got_cookies:
+ return
+
+ async with self.request(scrape_item.url, impersonate=True):
+ pass
+ cookies = self.manager.client_manager.cookies.filter_cookies(PRIMARY_URL)
+ self.got_cookies = bool(cookies)
diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.py
index 1f1acce9d..a75857bb7 100644
--- a/cyberdrop_dl/crawlers/sendvid.py
+++ b/cyberdrop_dl/crawlers/sendvid.py
@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
PRIMARY_URL = AbsoluteHttpURL("https://sendvid.com/")
@@ -38,8 +36,7 @@ class SendVidCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
title = css.select_one_get_text(soup, TITLE_SELECTOR)
try:
diff --git a/cyberdrop_dl/crawlers/sex_dot_com.py b/cyberdrop_dl/crawlers/sex_dot_com.py
index 27fe6e556..cf8c9a643 100644
--- a/cyberdrop_dl/crawlers/sex_dot_com.py
+++ b/cyberdrop_dl/crawlers/sex_dot_com.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import itertools
from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
@@ -14,6 +15,7 @@ if TYPE_CHECKING:
PRIMARY_URL = AbsoluteHttpURL("https://sex.com")
API_URL = AbsoluteHttpURL("https://iframe.sex.com/api/")
+_SHORTS_URL = AbsoluteHttpURL("https://sex.com/en/shorts")
class SexDotComCrawler(Crawler):
@@ -30,18 +32,14 @@ class SexDotComCrawler(Crawler):
return await self.profile(scrape_item)
raise ValueError
- async def shorts_profile_paginator(self, scrape_item: ScrapeItem) -> AsyncGenerator[dict]:
+ async def shorts_profile_paginator(self, scrape_item: ScrapeItem) -> AsyncGenerator[dict[str, Any]]:
username = scrape_item.url.parts[3]
- page = 1
- while True:
- posts_api_url = API_URL / "feed" / "listUserItems"
- query = {"pageSize": 300, "pageNumber": page, "visibility": "public", "username": username}
- posts_api_url = posts_api_url.with_query(query)
- async with self.request_limiter:
- json_data = await self.client.get_json(self.DOMAIN, posts_api_url)
+ api_url = (API_URL / "feed" / "listUserItems").with_query(pageSize=300, visibility="public", username=username)
+ for page in itertools.count(1):
+ json_data: dict[str, Any] = await self.request_json(api_url.with_query(pageNumber=page))
if scrape_item.album_id is None:
- user_id = json_data["page"]["items"][0]["media"]["user"]["userUid"]
+ user_id: str = json_data["page"]["items"][0]["media"]["user"]["userUid"]
title = self.create_title(username, user_id)
scrape_item.setup_as_profile(title, album_id=user_id)
@@ -49,32 +47,28 @@ class SexDotComCrawler(Crawler):
if not json_data["page"]["pageInfo"]["hasNextPage"]:
break
- page += 1
async def get_media(self, scrape_item: ScrapeItem) -> dict[str, Any]:
"""Gets media from its relative URL."""
relative_url = "/".join(scrape_item.url.parts[3:])
- data_url = API_URL / "media" / "getMedia"
- data_url = data_url.with_query(relativeUrl=relative_url)
- async with self.request_limiter:
- json_data = await self.client.get_json(self.DOMAIN, data_url)
- return json_data["media"]
+ data_url = (API_URL / "media" / "getMedia").with_query(relativeUrl=relative_url)
+ return (await self.request_json(data_url))["media"]
@error_handling_wrapper
async def handle_media(self, scrape_item: ScrapeItem, item: dict[str, Any] | None) -> None:
real_item = item or await self.get_media(scrape_item)
relative_url = real_item["relativeUrl"]
- canonical_url = AbsoluteHttpURL("https://sex.com/en/shorts") / relative_url
+ canonical_url = _SHORTS_URL / relative_url
if await self.check_complete_from_referer(canonical_url):
return
fileType: str = real_item.get("fileType") or real_item["mediaType"]
if fileType.startswith("image"):
- media_url = AbsoluteHttpURL(real_item["fullPath"]).with_query(optimizer="image", width=1200)
+ media_url = self.parse_url(real_item["fullPath"]).with_query(optimizer="image", width=1200)
filename, ext = f"{real_item['pictureUid']}.jpg", "jpg"
elif fileType.startswith("video"):
- media_url = AbsoluteHttpURL(real_item["sources"][0]["fullPath"])
+ media_url = self.parse_url(real_item["sources"][0]["fullPath"])
filename, ext = self.get_filename_and_ext(media_url.name)
else:
return
diff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.py
index 085d5e175..9b47a572d 100644
--- a/cyberdrop_dl/crawlers/spankbang.py
+++ b/cyberdrop_dl/crawlers/spankbang.py
@@ -89,8 +89,7 @@ class SpankBangCrawler(Crawler):
title: str = ""
for page in itertools.count(1):
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, page_url)
+ soup = await self.request_soup(page_url, impersonate=True)
# Get full playlist info + title from the soup
playlist = PlaylistInfo.from_url(page_url, soup)
@@ -102,7 +101,7 @@ class SpankBangCrawler(Crawler):
for _, new_scrape_item in self.iter_children(scrape_item, soup, VIDEOS_SELECTOR, results=results):
n_videos += 1
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
if n_videos < 100:
break
@@ -117,9 +116,7 @@ class SpankBangCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, scrape_item.url)
-
+ soup = await self.request_soup(scrape_item.url, impersonate=True)
was_removed = soup.select_one(VIDEO_REMOVED_SELECTOR)
if was_removed or "This video is no longer available" in soup.text:
raise ScrapeError(410)
diff --git a/cyberdrop_dl/crawlers/streamable.py b/cyberdrop_dl/crawlers/streamable.py
index aaa944b2e..c2575ad23 100644
--- a/cyberdrop_dl/crawlers/streamable.py
+++ b/cyberdrop_dl/crawlers/streamable.py
@@ -42,8 +42,7 @@ class StreamableCrawler(Crawler):
return
ajax_url = AJAX_ENTRYPOINT / video_id
- async with self.request_limiter:
- json_resp: dict[str, Any] = await self.client.get_json(self.DOMAIN, ajax_url)
+ json_resp: dict[str, Any] = await self.request_json(ajax_url)
status: int = json_resp["status"]
if status != STATUS_OK:
diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.py
index 493261e89..232c435fa 100644
--- a/cyberdrop_dl/crawlers/tokyomotion.py
+++ b/cyberdrop_dl/crawlers/tokyomotion.py
@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
PRIMARY_URL = AbsoluteHttpURL("https://www.tokyomotion.net")
@@ -82,8 +80,7 @@ class TokioMotionCrawler(Crawler):
return
video_id = scrape_item.url.parts[2]
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
src = soup.select_one(_SELECTORS.VIDEO_SRC)
if not src:
@@ -104,8 +101,7 @@ class TokioMotionCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
img = soup.select_one(_SELECTORS.IMAGE)
if not img:
@@ -205,8 +201,7 @@ class TokioMotionCrawler(Crawler):
return "favorite"
if "album" in scrape_item.url.parts and len(scrape_item.url.parts) > 3:
return scrape_item.url.parts[3]
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
return css.select_one_get_text(soup, _SELECTORS.ALBUM_TITLE)
def add_user_title(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/crawlers/toonily.py b/cyberdrop_dl/crawlers/toonily.py
index b3c89864c..115f4c36e 100644
--- a/cyberdrop_dl/crawlers/toonily.py
+++ b/cyberdrop_dl/crawlers/toonily.py
@@ -37,8 +37,7 @@ class ToonilyCrawler(Crawler):
@error_handling_wrapper
async def series(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
title_tag = css.select_one(soup, Selector.SERIES_TITLE)
css.decompose(title_tag, "*")
@@ -49,8 +48,7 @@ class ToonilyCrawler(Crawler):
@error_handling_wrapper
async def chapter(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
series_name, chapter_title = css.select_one_get_text(soup, Selector.CHAPTER_TITLE).split(" - ", 1)
if scrape_item.type != FILE_HOST_PROFILE:
diff --git a/cyberdrop_dl/crawlers/twpornstars.py b/cyberdrop_dl/crawlers/twpornstars.py
index cc76ce17e..550103ea6 100644
--- a/cyberdrop_dl/crawlers/twpornstars.py
+++ b/cyberdrop_dl/crawlers/twpornstars.py
@@ -9,8 +9,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.crawlers.crawler import SupportedDomains
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -59,8 +57,7 @@ class TwPornstarsCrawler(TwimgCrawler):
async def media(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
if url := soup.select_one(_SELECTORS.PHOTO):
url = self.parse_url(css.get_attr(url, "src").replace(":large", ""))
await self.photo(scrape_item, url)
@@ -85,4 +82,4 @@ class TwPornstarsCrawler(TwimgCrawler):
scrape_item.setup_as_album(title)
title_created = True
for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.THUMBS):
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
diff --git a/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py b/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py
index 26bc06d9e..055db45d0 100644
--- a/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py
+++ b/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py
@@ -132,9 +132,7 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801
scrape_item.add_children()
async def get_xml(self, url: AbsoluteHttpURL) -> ElementTree.Element[str]:
- async with self.request_limiter:
- text = await self.client.get_text(self.DOMAIN, url)
- root_xml = ElementTree.XML(text)
+ root_xml = ElementTree.XML(await self.request_text(url))
if error := root_xml.find("error"):
details = error.attrib["details"]
error_code = 403 if error.attrib["type"] == "permissions" and "unknown" not in details.casefold() else 422
diff --git a/cyberdrop_dl/crawlers/vbulletin/vipergirls.py b/cyberdrop_dl/crawlers/vbulletin/vipergirls.py
index d5819590b..6a9b8e709 100644
--- a/cyberdrop_dl/crawlers/vbulletin/vipergirls.py
+++ b/cyberdrop_dl/crawlers/vbulletin/vipergirls.py
@@ -2,8 +2,6 @@ from __future__ import annotations
from typing import ClassVar
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.vbulletin._vbulletin import vBulletinCrawler
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -16,7 +14,4 @@ class ViperGirlsCrawler(vBulletinCrawler):
FOLDER_DOMAIN = "ViperGirls"
SUPPORTED_DOMAINS: ClassVar = "viper.click", "vipergirls.to"
VBULLETIN_API_ENDPOINT: ClassVar = AbsoluteHttpURL("https://viper.click/vr.php")
-
- def __post_init__(self) -> None:
- super().__post_init__()
- self.request_limiter = AsyncLimiter(4, 1)
+ _RATE_LIMIT = 4, 1
diff --git a/cyberdrop_dl/crawlers/vipr_dot_im.py b/cyberdrop_dl/crawlers/vipr_dot_im.py
index 6589944fd..4a4c2ab3e 100644
--- a/cyberdrop_dl/crawlers/vipr_dot_im.py
+++ b/cyberdrop_dl/crawlers/vipr_dot_im.py
@@ -34,8 +34,7 @@ class ViprImCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
link_str: str = css.select_one_get_attr(soup, IMG_SELECTOR, "src")
link = self.parse_url(link_str)
@@ -44,7 +43,7 @@ class ViprImCrawler(Crawler):
async def thumbnail(self, scrape_item: ScrapeItem) -> None:
scrape_item.url = self.get_canonical_url(scrape_item.url)
- self.manager.task_group.create_task(self.run(scrape_item))
+ self.create_task(self.run(scrape_item))
def get_canonical_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
return PRIMARY_URL / get_image_id(url)
diff --git a/cyberdrop_dl/crawlers/wetransfer.py b/cyberdrop_dl/crawlers/wetransfer.py
index 7a927e3e5..ed4af904e 100644
--- a/cyberdrop_dl/crawlers/wetransfer.py
+++ b/cyberdrop_dl/crawlers/wetransfer.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import json
from dataclasses import dataclass
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -31,39 +31,26 @@ class WeTransferCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
if "download." in scrape_item.url.host:
# We can download but db entry will not have a canonical URL
- return await self.direct_link(scrape_item, scrape_item.url)
+ return await self.direct_file(scrape_item)
await self.file(scrape_item)
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.host == "we.tl":
- scrape_item.url = await self.get_final_url(scrape_item)
+ scrape_item.url = await self._get_redirect_url(scrape_item.url)
file_info = get_file_info(scrape_item.url)
if await self.check_complete_from_referer(file_info.download_url):
return
- async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.DOMAIN, file_info.download_url, json=file_info.json)
-
+ json_resp: dict[str, Any] = await self.request_json(file_info.download_url, method="POST", json=file_info.json)
link_str: str | None = json_resp.get("direct_link")
if not link_str:
code, msg = parse_error(json_resp)
raise ScrapeError(code, message=msg)
link = self.parse_url(link_str)
- await self.direct_link(scrape_item, link)
-
- @error_handling_wrapper
- async def direct_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:
- filename, ext = self.get_filename_and_ext(link.name)
- await self.handle_file(link, scrape_item, filename, ext)
-
- async def get_final_url(self, scrape_item: ScrapeItem) -> AbsoluteHttpURL:
- async with self.request_limiter:
- headers = await self.client.get_head(self.DOMAIN, scrape_item.url)
-
- return self.parse_url(headers["location"])
+ await self.direct_file(scrape_item, link)
@dataclass(frozen=True, order=True)
diff --git a/cyberdrop_dl/crawlers/wordpress/_wordpress.py b/cyberdrop_dl/crawlers/wordpress/_wordpress.py
index b9434c99a..b4a0d92c4 100644
--- a/cyberdrop_dl/crawlers/wordpress/_wordpress.py
+++ b/cyberdrop_dl/crawlers/wordpress/_wordpress.py
@@ -13,7 +13,6 @@ import re
from abc import abstractmethod
from typing import TYPE_CHECKING, ClassVar, TypeVar, final
-from aiolimiter import AsyncLimiter
from bs4 import BeautifulSoup
from pydantic import BaseModel
@@ -79,15 +78,13 @@ class WordPressBaseCrawler(Crawler, is_abc=True):
DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {id} - {title}"
WP_USE_REGEX: ClassVar = True
SUPPORTS_THREAD_RECURSION: ClassVar = False
+ _RATE_LIMIT = 3, 1
def __init_subclass__(cls, **kwargs) -> None:
super().__init_subclass__(**kwargs)
assert cls.fetch is WordPressBaseCrawler.fetch
assert cls.fetch_with_date_range is WordPressBaseCrawler.fetch_with_date_range
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(3, 1)
-
@final
async def fetch(self, scrape_item: ScrapeItem) -> None:
date_range = QueryDatetimeRange.from_url(scrape_item.url)
@@ -233,8 +230,7 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):
await self.filter_post(scrape_item, post)
async def __make_request(self, model_cls: type[_ModelT], api_url: AbsoluteHttpURL) -> _ModelT:
- async with self.request_limiter:
- json_text = await self.client.get_text(self.DOMAIN, api_url)
+ json_text = await self.request_text(api_url)
return model_cls.model_validate_json(json_text)
async def __handle_collection(
@@ -264,8 +260,7 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):
class WordPressHTMLCrawler(WordPressBaseCrawler, is_generic=True):
async def __make_request(self, api_url: AbsoluteHttpURL) -> BeautifulSoup:
- async with self.request_limiter:
- return await self.client.get_soup(self.DOMAIN, api_url)
+ return await self.request_soup(api_url)
@error_handling_wrapper
async def category_or_tag(
@@ -320,7 +315,7 @@ class WordPressHTMLCrawler(WordPressBaseCrawler, is_generic=True):
f"Skipping post {new_scrape_item.url} as it is out of date range. Post date: {date_from_path}"
)
continue
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self.create_task(self.run(new_scrape_item))
def _match_date_from_path(url_parts: tuple[str, ...]) -> datetime.datetime | None:
diff --git a/cyberdrop_dl/crawlers/xbunkr.py b/cyberdrop_dl/crawlers/xbunkr.py
index af173d4f5..cdbf63d86 100644
--- a/cyberdrop_dl/crawlers/xbunkr.py
+++ b/cyberdrop_dl/crawlers/xbunkr.py
@@ -8,8 +8,6 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -31,8 +29,7 @@ class XBunkrCrawler(Crawler):
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
album_id = scrape_item.url.parts[2]
title = self.create_title(css.select_one_get_text(soup, TITLE_SELECTOR), album_id)
diff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.py
index df4caf1fe..63c13d236 100644
--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py
+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py
@@ -17,9 +17,7 @@ class F95ZoneCrawler(XenforoCrawler):
@error_handling_wrapper
async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
- async with self.request_limiter:
- json_resp = await self.client.post_data(self.DOMAIN, link, data=_confirmation_data)
-
+ json_resp = await self.request_json(link, method="POST", data=_confirmation_data)
if json_resp["status"] == "ok":
return self.parse_url(json_resp["msg"])
diff --git a/cyberdrop_dl/crawlers/xenforo/simpcity.py b/cyberdrop_dl/crawlers/xenforo/simpcity.py
index 259b34bc7..0b2d44443 100644
--- a/cyberdrop_dl/crawlers/xenforo/simpcity.py
+++ b/cyberdrop_dl/crawlers/xenforo/simpcity.py
@@ -2,8 +2,6 @@ from __future__ import annotations
from typing import ClassVar
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from .xenforo import XenforoCrawler
@@ -17,7 +15,4 @@ class SimpCityCrawler(XenforoCrawler):
login_required = False
IGNORE_EMBEDED_IMAGES_SRC = False
OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("simpcity.su",)
-
- def __post_init__(self) -> None:
- super().__post_init__()
- self.request_limiter = AsyncLimiter(1, 10)
+ _RATE_LIMIT = 1, 20
diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.py
index 2dfebdf95..dae4dab15 100644
--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py
+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py
@@ -124,7 +124,13 @@ class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):
attempt += 1
await asyncio.sleep(wait_time)
data = parse_login_form(text) | credentials
- _ = await self.client._post_data(self.DOMAIN, login_url / "login", data=data, cache_disabled=True)
+ async with self.request(
+ login_url / "login",
+ method="POST",
+ data=data,
+ cache_disabled=True,
+ ):
+ pass
await asyncio.sleep(wait_time)
text, logged_in = await self.check_login_with_request(login_url)
if logged_in:
@@ -132,6 +138,7 @@ class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):
return
except TimeoutError:
continue
+
msg = f"Failed to login on {self.FOLDER_DOMAIN} after {retries} attempts"
raise LoginError(message=msg)
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.py
index 4b46f415e..81f24b441 100644
--- a/cyberdrop_dl/crawlers/xhamster.py
+++ b/cyberdrop_dl/crawlers/xhamster.py
@@ -6,8 +6,6 @@ import itertools
import json
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_url, xor_decrypt
@@ -53,9 +51,9 @@ class XhamsterCrawler(Crawler):
NEXT_PAGE_SELECTOR = Selector.NEXT_PAGE
DOMAIN = "xhamster"
FOLDER_DOMAIN = "xHamster"
+ _RATE_LIMIT = 4, 1
def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(4, 1)
self._seen_hosts: set[str] = set()
def _disable_ai_title_translations(self, url: AbsoluteHttpURL) -> None:
@@ -208,9 +206,7 @@ class XhamsterCrawler(Crawler):
async def _get_window_initials(self, url: AbsoluteHttpURL) -> dict[str, Any]:
self._disable_ai_title_translations(url)
- async with self.request_limiter:
- content = await self.client.get_text(self.DOMAIN, url)
-
+ content = await self.request_text(url)
initials = get_text_between(content, "window.initials=", ";</script>")
return json.loads(initials)
diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.py
index 4ff07440d..a2a4a566a 100644
--- a/cyberdrop_dl/crawlers/xvideos.py
+++ b/cyberdrop_dl/crawlers/xvideos.py
@@ -179,28 +179,35 @@ class XVideosCrawler(Crawler):
self.create_task(self.direct_file(scrape_item, src))
async def _get_soup(self, url: AbsoluteHttpURL) -> BeautifulSoup:
- async with self._domain_locks[url.host]:
- if url.host not in self._seen_domains:
- await self._disable_auto_translated_titles(url.origin())
- self._seen_domains.add(url.host)
+ if url.host not in self._seen_domains:
+ async with self._domain_locks[url.host]:
+ if url.host not in self._seen_domains:
+ await self._disable_auto_translated_titles(url.origin())
+ self._seen_domains.add(url.host)
- async with self.request_limiter:
- return await self.client.get_soup(self.DOMAIN, url, self._headers)
+ return await self.request_soup(url, headers=self._headers)
async def _disable_auto_translated_titles(self, origin: AbsoluteHttpURL) -> None:
- async with self.request_limiter:
- await self.client._get(self.DOMAIN, origin / "change-language/en", self._headers)
- json_resp = await self.client.post_data(
- self.DOMAIN, origin / "account/feature-disabled", self._headers, data={"featureid": "at"}
- )
+ async with self.request(origin / "change-language/en", headers=self._headers):
+ pass
+
+ json_resp: dict[str, Any] = await self.request_json(
+ origin / "account/feature-disabled",
+ method="POST",
+ headers=self._headers,
+ data={"featureid": "at"},
+ )
if json_resp["code"] != 0:
self.disabled = True
raise ScrapeError(json_resp["code"])
async def _iter_api_pages(self, scrape_item: ScrapeItem, api_url: AbsoluteHttpURL, new_part: str) -> None:
for page in itertools.count(0):
- async with self.request_limiter:
- json_resp: dict[str, Any] = await self.client.post_data(self.DOMAIN, api_url / str(page), self._headers)
+ json_resp: dict[str, Any] = await self.request_json(
+ api_url / str(page),
+ method="POST",
+ headers=self._headers,
+ )
if json_resp["code"] != 0:
raise ScrapeError(json_resp["code"])
diff --git a/cyberdrop_dl/crawlers/xxxbunker.py b/cyberdrop_dl/crawlers/xxxbunker.py
index 31ccdbe8c..1db7c5cf1 100644
--- a/cyberdrop_dl/crawlers/xxxbunker.py
+++ b/cyberdrop_dl/crawlers/xxxbunker.py
@@ -2,8 +2,6 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
@@ -34,9 +32,7 @@ class XXXBunkerCrawler(Crawler):
FOLDER_DOMAIN: ClassVar[str] = "XXXBunker"
NEXT_PAGE_SELECTOR = Selector.NEXT_PAGE
_DOWNLOAD_SLOTS: ClassVar[int | None] = 2
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(1, 6)
+ _RATE_LIMIT = 1, 6
async def async_startup(self) -> None:
self.update_cookies({"ageconfirm": "True"})
@@ -57,16 +53,11 @@ class XXXBunkerCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
- async with self.request_limiter:
- soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
-
+ soup = await self.request_soup(scrape_item.url)
_check_video_is_available(soup)
title = open_graph.title(soup)
iframe_url = self.parse_url(css.select_one_get_attr(soup, Selector.VIDEO_IFRAME, "data-src"))
-
- async with self.request_limiter:
- iframe_soup = await self.client.get_soup(self.DOMAIN, iframe_url)
-
+ iframe_soup = await self.request_soup(iframe_url)
src = self.parse_url(css.select_one_get_attr(iframe_soup, "source", "src"))
video_id = iframe_url.name
custom_filename = self.create_custom_filename(title, ".mp4", file_id=video_id)
diff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.py
index 6b9b53412..f08881e08 100644
--- a/cyberdrop_dl/crawlers/yandex_disk.py
+++ b/cyberdrop_dl/crawlers/yandex_disk.py
@@ -71,9 +71,7 @@ class YandexDiskCrawler(Crawler):
return
async with self.request_context():
- soup: BeautifulSoup = await self.client.get_soup(
- self.DOMAIN, scrape_item.url, _DEFAULT_HEADERS, cache_disabled=True
- )
+ soup = await self.request_soup(scrape_item.url, headers=_DEFAULT_HEADERS, cache_disabled=True)
item_info = get_item_info(soup)
assert is_single_item(item_info)
@@ -90,9 +88,7 @@ class YandexDiskCrawler(Crawler):
scrape_item.url = canonical_url
async with self.request_context():
- soup: BeautifulSoup = await self.client.get_soup(
- self.DOMAIN, scrape_item.url, _DEFAULT_HEADERS, cache_disabled=True
- )
+ soup = await self.request_soup(scrape_item.url, headers=_DEFAULT_HEADERS, cache_disabled=True)
item_info = get_item_info(soup)
del soup
@@ -123,8 +119,7 @@ class YandexDiskCrawler(Crawler):
@contextlib.asynccontextmanager
async def request_context(self) -> AsyncGenerator[None]:
try:
- async with self.request_limiter:
- yield
+ yield
except DownloadError as e:
if e.status in (400, 403):
raise DDOSGuardError from None
@@ -146,7 +141,12 @@ class YandexDiskCrawler(Crawler):
api_url = DOWNLOAD_API_ENTRYPOINT.with_host(scrape_item.url.host)
async with self.request_context():
- json_resp: dict[str, Any] = await self.client.post_data(self.DOMAIN, api_url, headers, data=file.post_data)
+ json_resp: dict[str, Any] = await self.request_json(
+ api_url,
+ method="POST",
+ headers=headers,
+ data=file.post_data,
+ )
new_sk = json_resp.get("new_sk")
if new_sk:
@@ -155,9 +155,7 @@ class YandexDiskCrawler(Crawler):
error = json_resp.get("error")
if error:
- # The error format is dynamic but they are short
- # We can log them to the main file
- raise ScrapeError(422, message=json.dumps(json_resp))
+ raise ScrapeError(422, message=json.dumps(json_resp)[:50])
self.log_debug(json_resp)
scrape_item.possible_datetime = file.modified
diff --git a/cyberdrop_dl/crawlers/youjizz.py b/cyberdrop_dl/crawlers/youjizz.py
index ab3a08cd5..5b50e346f 100644
--- a/cyberdrop_dl/crawlers/youjizz.py
+++ b/cyberdrop_dl/crawlers/youjizz.py
@@ -60,8 +60,7 @@ class YouJizzCrawler(Crawler):
if await self.check_complete_from_referer(canonical_url):
return
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.request_soup(scrape_item.url)
scrape_item.url = canonical_url
video = _parse_video(soup)
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index e1415a6e6..505bc3a38 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -477,7 +477,7 @@ class MegaApi:
self.root_id: str = ""
self.inbox_id: str = ""
self.trashbin_id: str = ""
- self.request_limiter = AsyncLimiter(100, 60)
+ self._limiter = AsyncLimiter(100, 60)
self._files = {}
self.shared_keys: SharedkeysDict
@@ -498,7 +498,7 @@ class MegaApi:
else:
data: list[AnyDict] = data_input
- async with self.request_limiter, self.session.disabled():
+ async with self._limiter, self.session.disabled():
response = await self.session.post(
self.entrypoint, params=params, json=data, timeout=self.timeout, headers=self.default_headers
)
@@ -513,7 +513,7 @@ class MegaApi:
log("[MegaNZ] Solving xhashcash login challenge, this could take a few seconds...")
xhashcash_token = await generate_hashcash_token(xhashcash_challenge)
headers = self.default_headers | {"X-Hashcash": xhashcash_token}
- async with self.request_limiter, self.session.disabled():
+ async with self._limiter, self.session.disabled():
response = await self.session.post(
self.entrypoint, params=params, json=data, timeout=self.timeout, headers=headers
)
diff --git a/cyberdrop_dl/managers/cache_manager.py b/cyberdrop_dl/managers/cache_manager.py
index 1844a932c..875b5594f 100644
--- a/cyberdrop_dl/managers/cache_manager.py
+++ b/cyberdrop_dl/managers/cache_manager.py
@@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Any
from aiohttp_client_cache import CacheBackend, SQLiteBackend
from cyberdrop_dl import __version__ as current_version
-from cyberdrop_dl.data_structures.supported_domains import SUPPORTED_FORUMS, SUPPORTED_WEBSITES
from cyberdrop_dl.scraper.filters import cache_filter_fn
from cyberdrop_dl.utils import yaml
@@ -41,6 +40,8 @@ class CacheManager:
self._cache = yaml.load(self.cache_file)
def load_request_cache(self) -> None:
+ from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_WEBSITES
+
rate_limiting_options = self.manager.config_manager.global_settings_data.rate_limiting_options
urls_expire_after = {
"*.simpcity.su": rate_limiting_options.file_host_cache_expire_after,
diff --git a/cyberdrop_dl/data_structures/supported_domains.py b/cyberdrop_dl/supported_domains.py
similarity index 100%
rename from cyberdrop_dl/data_structures/supported_domains.py
rename to cyberdrop_dl/supported_domains.py
diff --git a/cyberdrop_dl/ui/prompts/user_prompts.py b/cyberdrop_dl/ui/prompts/user_prompts.py
index 4e61d7847..dc66c1cf6 100644
--- a/cyberdrop_dl/ui/prompts/user_prompts.py
+++ b/cyberdrop_dl/ui/prompts/user_prompts.py
@@ -16,11 +16,6 @@ from rich.console import Console
from cyberdrop_dl import __version__
from cyberdrop_dl.constants import BROWSERS, RESERVED_CONFIG_NAMES
-from cyberdrop_dl.data_structures.supported_domains import (
- SUPPORTED_FORUMS,
- SUPPORTED_SITES_DOMAINS,
- SUPPORTED_WEBSITES,
-)
from cyberdrop_dl.ui.prompts import basic_prompts
from cyberdrop_dl.ui.prompts.defaults import ALL_CHOICE, DONE_CHOICE, EXIT_CHOICE
from cyberdrop_dl.utils.cookie_management import get_cookies_from_browsers
@@ -160,6 +155,8 @@ class DomainType(IntEnum):
def domains_prompt(*, domain_message: str = "Select site(s):") -> tuple[list[str], list[str]]:
"""Asks the user to select website(s) for cookie actions and cache actions."""
+ from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_SITES_DOMAINS, SUPPORTED_WEBSITES
+
OPTIONS = [["Forum", "File Host"], ["All Supported Websites"]]
choices = basic_prompts.create_choices(OPTIONS)
domain_type = basic_prompts.ask_choice(choices, message="Select category:")
@@ -200,6 +197,7 @@ def domains_prompt(*, domain_message: str = "Select site(s):") -> tuple[list[str
def extract_cookies(manager: Manager, *, dry_run: bool = False) -> None:
"""Asks the user to select browser(s) and domains(s) to import cookies from."""
+ from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_SITES_DOMAINS, SUPPORTED_WEBSITES
supported_forums, supported_websites = list(SUPPORTED_FORUMS.values()), list(SUPPORTED_WEBSITES.values())
domains, all_domains = domains_prompt(domain_message="Select site(s) to import cookies from:")
diff --git a/cyberdrop_dl/utils/cookie_management.py b/cyberdrop_dl/utils/cookie_management.py
index 5e6d5afc3..5e537dd4f 100644
--- a/cyberdrop_dl/utils/cookie_management.py
+++ b/cyberdrop_dl/utils/cookie_management.py
@@ -21,12 +21,6 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
-from cyberdrop_dl.data_structures.supported_domains import (
- SUPPORTED_FORUMS,
- SUPPORTED_SITES_DOMAINS,
- SUPPORTED_WEBSITES,
-)
-
P = ParamSpec("P")
R = TypeVar("R")
@@ -77,6 +71,8 @@ def get_cookies_from_browsers(manager: Manager, *, browser: BROWSERS, domains: l
:param domains: list of domains to filter cookies. If `None`, config `browser_cookies.sites` will be used
:return: A set with all the domains that actually had cookies
:raises BrowserCookieError: If there's any error while extracting cookies"""
+ from cyberdrop_dl.supported_domains import SUPPORTED_FORUMS, SUPPORTED_SITES_DOMAINS, SUPPORTED_WEBSITES
+
if domains == []:
msg = "No domains selected"
raise ValueError(msg)
@@ -164,7 +160,7 @@ async def read_netscape_files(cookie_files: list[Path]) -> AsyncIterable[tuple[s
log(f"Cookies for {simplified_domain} are expired", 30)
domains_seen.add(simplified_domain)
- simple_cookie = _make_simple_cookie(cookie, now)
+ simple_cookie = make_simple_cookie(cookie, now)
yield cookie.domain, simple_cookie
@@ -180,7 +176,7 @@ async def _read_netscape_file(file: Path) -> MozillaCookieJar | None:
return await asyncio.to_thread(read)
-def _make_simple_cookie(cookie: Cookie, now: float) -> SimpleCookie:
+def make_simple_cookie(cookie: Cookie, now: float) -> SimpleCookie:
simple_cookie = SimpleCookie()
assert cookie.value is not None
simple_cookie[cookie.name] = cookie.value
diff --git a/tests/crawlers/test_cases/xhamster.py b/tests/crawlers/test_cases/xhamster.py
index f5b758d3c..63d12906b 100644
--- a/tests/crawlers/test_cases/xhamster.py
+++ b/tests/crawlers/test_cases/xhamster.py
@@ -56,7 +56,7 @@ TEST_CASES = [
"datetime": 1372456260,
}
],
- 42,
+ 40,
),
(
"https://xhamster.com/creators/danielagomezz",
diff --git a/tests/crawlers/test_cases/xvideos.py b/tests/crawlers/test_cases/xvideos.py
index 1f468bb5c..69cb03e77 100644
--- a/tests/crawlers/test_cases/xvideos.py
+++ b/tests/crawlers/test_cases/xvideos.py
@@ -52,7 +52,7 @@ TEST_CASES = [
{
"url": "https://it.xvideos.com/video.uceupff0647/_",
"filename": "Beauty Gina Gerson Takes Monster Cock in Anal - Anal Orgasm [uceupff0647][1080p].mp4",
- "referer": "https://it.xvideos.com/video.uceupff0647/beauty_gina_gerson_takes_monster_cock_in_anal_-_anal_orgasm",
+ "referer": "https://it.xvideos.com/video.uceupff0647/_",
}
],
),