Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: type errors from ruff refactor (#1874)

NTFSvolume4f4a1fe462e7
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index 8159283af..3f2d3d89a 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -449,7 +449,9 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
for article in soup.select(self.SELECTORS.posts.article):
current_post = ForumPost.new(article, self.SELECTORS.posts)
continue_scraping, scrape_this_post = check_post_id(
- thread.post_id, current_post.id, self.scrape_single_forum_post
+ thread.post_id,
+ current_post.id,
+ scrape_single_forum_post=self.scrape_single_forum_post,
)
if scrape_this_post:
post_url = self.make_post_url(thread, current_post.id)
@@ -536,7 +538,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
@final
@error_handling_wrapper
async def process_child(self, scrape_item: ScrapeItem, link_str: str, *, embeds: bool = False) -> None:
- link_str_ = pre_process_child(link_str, embeds)
+ link_str_ = pre_process_child(link_str, embeds=embeds)
if not link_str_:
return None
link = await self.get_absolute_link(link_str_)
diff --git a/cyberdrop_dl/crawlers/_hls.py b/cyberdrop_dl/crawlers/_hls.py
index fcc874483..97c36a169 100644
--- a/cyberdrop_dl/crawlers/_hls.py
+++ b/cyberdrop_dl/crawlers/_hls.py
@@ -88,6 +88,6 @@ class SimpleHLSParser(HLSParser):
_session: aiohttp.ClientSession
@override
- async def request_text(self, url: AbsoluteHttpURL, /, headers: Mapping[str, str] | None = None) -> str:
+ async def request_text(self, url: AbsoluteHttpURL, /, *, headers: Mapping[str, str] | None = None) -> str:
async with self._session.get(url, headers=headers) as resp:
return await resp.text()
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index cff83a638..840e09fa7 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -175,6 +175,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
FOLDER_DOMAIN: ClassVar[str] = ""
DOMAIN: ClassVar[str]
PRIMARY_URL: ClassVar[AbsoluteHttpURL]
+ _FORUM: ClassVar[bool] = False
_RATE_LIMIT: ClassVar[RateLimit] = 25, 1
_DOWNLOAD_SLOTS: ClassVar[int | None] = None
@@ -598,7 +599,6 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
self,
filename: str,
*,
- forum: bool = False,
assume_ext: str | None = ".mp4",
mime_type: str | None = None,
) -> tuple[str, str]:
@@ -607,10 +607,10 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
If that fails, appends `assume_ext` and tries again, but only if the user had exclude_files_with_no_extension = `False`
"""
try:
- return get_filename_and_ext(filename, mime_type, xenforo=forum)
+ return get_filename_and_ext(filename, mime_type, xenforo=self._FORUM)
except NoExtensionError:
if self.allow_no_extension and assume_ext:
- return get_filename_and_ext(filename + assume_ext, mime_type, xenforo=forum)
+ return get_filename_and_ext(filename + assume_ext, mime_type, xenforo=self._FORUM)
raise
@final
@@ -799,12 +799,12 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
@final
@classmethod
- def parse_date(cls, date_or_datetime: str, /, format: str) -> dates.TimeStamp | None: # noqa: A002
+ def parse_date(cls, date_or_datetime: str, /, format: str) -> dates.TimeStamp: # noqa: A002
return dates.to_timestamp(dates.parse_format(date_or_datetime, format))
@final
@classmethod
- def parse_iso_date(cls, date_or_datetime: str, /) -> dates.TimeStamp | None:
+ def parse_iso_date(cls, date_or_datetime: str, /) -> dates.TimeStamp:
return dates.to_timestamp(dates.parse_iso(date_or_datetime))
async def _get_redirect_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
diff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.py
index 716b50f39..cd9b6ddf0 100644
--- a/cyberdrop_dl/crawlers/efukt.py
+++ b/cyberdrop_dl/crawlers/efukt.py
@@ -4,10 +4,8 @@ from pathlib import Path
from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
-from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper
-from cyberdrop_dl.utils.dates import to_timestamp
+from cyberdrop_dl.utils import css, dates, error_handling_wrapper
if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
@@ -67,10 +65,8 @@ class EfuktCrawler(Crawler):
soup = await self.request_soup(scrape_item.url)
date_str = css.select_text(soup, _SELECTORS.DATE).split(" ", 1)[-1]
- datetime = self._parse_date(date_str, "%m/%d/%y")
- if not datetime:
- raise ScrapeError(422)
- scrape_item.uploaded_at = to_timestamp(datetime)
+ date = dates.parse_format(date_str, "%m/%d/%y")
+ scrape_item.uploaded_at = dates.to_timestamp(date)
if is_image_or_gif(scrape_item.url):
link = self.parse_url(css.select(soup, _SELECTORS.IMAGE, "src"))
@@ -80,7 +76,7 @@ class EfuktCrawler(Crawler):
item_id = scrape_item.url.query.get("id") or scrape_item.url.name.partition("_")[0]
title = Path(css.select_text(soup, _SELECTORS.TITLE)).as_posix().replace("/", "-")
filename, ext = self.get_filename_and_ext(link.name)
- custom_filename = self.create_custom_filename(f"{datetime.date().isoformat()} {title}", ext, file_id=item_id)
+ custom_filename = self.create_custom_filename(f"{date.date().isoformat()} {title}", ext, file_id=item_id)
# Video links expire, but the path is always the same, only query params change
await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=link)
diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.py
index c4f9ba481..db38cef90 100644
--- a/cyberdrop_dl/crawlers/kemono.py
+++ b/cyberdrop_dl/crawlers/kemono.py
@@ -8,7 +8,7 @@ from collections import defaultdict
from collections.abc import Generator
from datetime import datetime # noqa: TC003
from pathlib import Path
-from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Concatenate, Literal, NamedTuple, ParamSpec
+from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Concatenate, NamedTuple, ParamSpec
from pydantic import BeforeValidator, Field
@@ -321,7 +321,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
@fallback_if_no_api
@error_handling_wrapper
- async def favorites(self, scrape_item: ScrapeItem, type_: Literal["post", "artist"]) -> None:
+ async def favorites(self, scrape_item: ScrapeItem, type_: str) -> None:
if not self.session_cookie:
msg = "No session cookie found in the config file, cannot scrape favorites"
raise ScrapeError(401, msg)
@@ -366,7 +366,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
if not server:
continue
- path = attach.get("path")
+ path = attach.get("path", "")
if previous_server := self.__known_attachment_servers.get(path):
if previous_server != server:
msg = (
diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.py
index c083c48d0..054a30948 100644
--- a/cyberdrop_dl/crawlers/onedrive.py
+++ b/cyberdrop_dl/crawlers/onedrive.py
@@ -67,7 +67,7 @@ class OneDriveFile(OneDriveItem):
download_url: AbsoluteHttpURL
@classmethod
- def from_api_response(cls, json_resp: dict, access_details: AccessDetails) -> Self:
+ def from_api_response(cls, json_resp: dict[str, Any], access_details: AccessDetails) -> Self:
info = parse_api_response(json_resp, access_details)
download_url_str = json_resp["@content.downloadUrl"]
info["download_url"] = AbsoluteHttpURL(download_url_str, encoded="%" in download_url_str)
@@ -79,7 +79,7 @@ class OneDriveFolder(OneDriveItem):
children: list[dict[str, Any]]
@classmethod
- def from_api_response(cls, json_resp: dict, access_details: AccessDetails) -> Self:
+ def from_api_response(cls, json_resp: dict[str, Any], access_details: AccessDetails) -> Self:
info = parse_api_response(json_resp, access_details)
info["children"] = json_resp["children"]
return cls(**info)
@@ -147,7 +147,7 @@ class OneDriveCrawler(Crawler):
raise ScrapeError(401)
api_url = create_api_url(access_details)
- json_resp: dict = await self.make_api_request(api_url)
+ json_resp: dict[str, Any] = await self.make_api_request(api_url)
if not is_folder(json_resp):
file = OneDriveFile.from_api_response(json_resp, access_details)
@@ -223,7 +223,7 @@ def is_folder(json_resp: dict[str, Any]) -> bool:
return bool(json_resp.get("folder"))
-def parse_api_response(json_resp: dict, access_details: AccessDetails) -> dict[str, Any]:
+def parse_api_response(json_resp: dict[str, Any], access_details: AccessDetails) -> dict[str, Any]:
web_url_str: str = json_resp["webUrl"]
item_id = json_resp["id"]
date_str = json_resp["fileSystemInfo"]["lastModifiedDateTime"]
diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.py
index dbf672b6c..41402c6bc 100644
--- a/cyberdrop_dl/crawlers/pornhub.py
+++ b/cyberdrop_dl/crawlers/pornhub.py
@@ -81,7 +81,7 @@ class Media(TypedDict):
width: int
format: Literal["hls", "mp4"]
videoUrl: str
- quality: str | list
+ quality: str | list[str]
class Format(NamedTuple):
diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.py
index 0bb896253..0d1b4be7f 100644
--- a/cyberdrop_dl/crawlers/rumble.py
+++ b/cyberdrop_dl/crawlers/rumble.py
@@ -158,7 +158,7 @@ class RumbleCrawler(Crawler):
_, ext = self.get_filename_and_ext(best_format.url.name)
video_name = self.create_custom_filename(video.title, ext, file_id=embed_id, resolution=best_format.resolution)
- scrape_item.uploaded_at = self.parse_iso_date(video["uploadDate"])
+ scrape_item.uploaded_at = self.parse_iso_date(video.upload_date)
scrape_item.url = video.url
self.create_task(
self.handle_file(
@@ -203,6 +203,6 @@ class RumbleCrawler(Crawler):
)
if hls_formats:
- hls_formats = await aio.map(resolve_m3u8, hls_formats)
+ hls_formats = await aio.map(resolve_m3u8, hls_formats, task_limit=10)
return max((*hls_formats, *other_formats))
diff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.py
index c46bbf0f7..7ed3505e9 100644
--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py
+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py
@@ -1,12 +1,15 @@
from __future__ import annotations
-from typing import ClassVar
+from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import error_handling_wrapper
from .xenforo import XenforoCrawler
+if TYPE_CHECKING:
+ import yarl
+
_confirmation_data = ({"xhr": "1", "download": "1"},)
@@ -30,8 +33,15 @@ class F95ZoneCrawler(XenforoCrawler):
return url.with_path(url.path.replace("/thumb/", ""))
@classmethod
- def parse_url(cls, link: str) -> AbsoluteHttpURL:
- url = super().parse_url(link)
+ def parse_url(
+ cls,
+ url: yarl.URL | str,
+ /,
+ relative_to: AbsoluteHttpURL | None = None,
+ *,
+ trim: bool | None = None,
+ ) -> AbsoluteHttpURL:
+ url = super().parse_url(url, relative_to, trim=trim)
if cls.is_thumbnail(url):
return cls.thumbnail_to_img(url)
return url
diff --git a/cyberdrop_dl/crawlers/xenforo/titsintops.py b/cyberdrop_dl/crawlers/xenforo/titsintops.py
index 61eeceb97..182a99944 100644
--- a/cyberdrop_dl/crawlers/xenforo/titsintops.py
+++ b/cyberdrop_dl/crawlers/xenforo/titsintops.py
@@ -8,6 +8,7 @@ from cyberdrop_dl.utils import css
from .xenforo import XenforoCrawler
if TYPE_CHECKING:
+ import yarl
from bs4 import Tag
@@ -17,8 +18,15 @@ class TitsInTopsCrawler(XenforoCrawler):
FOLDER_DOMAIN: ClassVar[str] = "TitsInTops"
@classmethod
- def parse_url(cls, link: str) -> AbsoluteHttpURL:
- return super().parse_url(_query_to_path(link))
+ def parse_url(
+ cls,
+ url: yarl.URL | str,
+ /,
+ relative_to: AbsoluteHttpURL | None = None,
+ *,
+ trim: bool | None = None,
+ ) -> AbsoluteHttpURL:
+ return super().parse_url(_query_to_path(str(url)), relative_to, trim=trim)
def is_username_or_attachment(self, link_obj: Tag) -> bool:
text = css.text(link_obj)
diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.py
index 861f4e8d0..42155fcd7 100644
--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py
+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py
@@ -74,15 +74,11 @@ class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):
PAGE_URL_PART_NAME: ClassVar[str] = "page"
IGNORE_EMBEDED_IMAGES_SRC: ClassVar[bool] = True
LOGIN_USER_COOKIE_NAME: ClassVar[str] = "xf_user"
+ _FORUM: ClassVar[bool] = True
# Attachments hosts should technically be defined on each specific Crawler, but they do no harm here
ATTACHMENT_HOSTS = "smgmedia", "attachments.f95zone"
login_required = True
- def get_filename_and_ext(self, filename: str) -> tuple[str, str]:
- # The `forum` keyword is misleading now. It only works for Xenforo sites, not every forum
- # TODO: Change `forum` parameter to `xenforo`
- return super().get_filename_and_ext(filename, forum=True)
-
@error_handling_wrapper
async def xf_login(self, login_url: AbsoluteHttpURL, session_cookie: str, username: str, password: str) -> None:
"""Logic to login as a Xenforo user
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.py
index b70c84a58..e1c9fe16d 100644
--- a/cyberdrop_dl/crawlers/xhamster.py
+++ b/cyberdrop_dl/crawlers/xhamster.py
@@ -190,7 +190,7 @@ class XhamsterCrawler(Crawler):
initials = await self._get_window_initials(next_page_url)
images = initials["photosGalleryModel"]["photos"]
- def _handle_img(self, scrape_item: ScrapeItem, img: dict[str, Any], results: dict[str, int]) -> None:
+ def _handle_img(self, scrape_item: ScrapeItem, img: dict[str, Any], results: dict[str, bool]) -> None:
src, page_url = self.parse_url(img["imageURL"]), self.parse_url(img["pageURL"])
if self.check_album_results(src, results):
return
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 619964b3c..9bee3dcc1 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -53,7 +53,7 @@ def select_text(tag: Tag, selector: str, *, strip: bool = True, decompose: str |
if decompose:
for trash in inner_tag.select(decompose):
trash.decompose()
- return text(inner_tag, strip)
+ return text(inner_tag, strip=strip)
def attr_or_none(tag: Tag, attribute: str) -> str | None: