Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: general type errors (#1084)

NTFSvolume06bf602be563
diff --git a/cyberdrop_dl/crawlers/_kemono_base.py b/cyberdrop_dl/crawlers/_kemono_base.py
index 505440e38..f7684d244 100644
--- a/cyberdrop_dl/crawlers/_kemono_base.py
+++ b/cyberdrop_dl/crawlers/_kemono_base.py
@@ -24,7 +24,6 @@ if TYPE_CHECKING:
from aiohttp_client_cache.response import AnyResponse
from bs4 import BeautifulSoup
- from yarl import URL
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
@@ -62,7 +61,7 @@ class UserURL(NamedTuple):
post_id: str | None = None
@staticmethod
- def parse(url: URL) -> UserURL:
+ def parse(url: AbsoluteHttpURL) -> UserURL:
if (n_parts := len(url.parts)) > 3:
post_id = url.parts[5] if n_parts > 5 else None
return UserURL(url.parts[1], url.parts[3], post_id)
@@ -74,7 +73,7 @@ class UserPostURL(UserURL):
post_id: str
@staticmethod
- def parse(url: URL) -> UserPostURL:
+ def parse(url: AbsoluteHttpURL) -> UserPostURL:
result = UserURL.parse(url)
assert result.post_id, "Individual posts must have a post_id"
return cast("UserPostURL", result)
@@ -87,7 +86,7 @@ class DiscordURL(NamedTuple):
channel_id: str | None = None # Only present for individual channels URLs
@staticmethod
- def parse(url: URL) -> DiscordURL:
+ def parse(url: AbsoluteHttpURL) -> DiscordURL:
return DiscordURL(*url.parts[3:5])
@@ -459,7 +458,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
"""~~~~~~~~ PRIVATE METHODS, should never be overriden ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
- async def __handle_post(self, scrape_item: ScrapeItem, post: Post):
+ async def __handle_post(self, scrape_item: ScrapeItem, post: Post) -> None:
# Process files if the post was generated from an API call
for file in post.all_files:
file_url = self.__make_file_url(file)
@@ -473,12 +472,12 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
self._handle_post_content(scrape_item, post)
- def __make_file_url(self, file: File) -> URL:
+ def __make_file_url(self, file: File) -> AbsoluteHttpURL:
server = self.__known_attachment_servers.get(file["path"], "")
url = server + f"/data{file['path']}"
return self.parse_url(url).with_query(f=file["name"])
- def __make_api_url_w_offset(self, path: str, og_url: URL) -> URL:
+ def __make_api_url_w_offset(self, path: str, og_url: AbsoluteHttpURL) -> AbsoluteHttpURL:
api_url = self.API_ENTRYPOINT / path
offset = int(og_url.query.get("o", 0))
if query := og_url.query.get("q"):
@@ -519,7 +518,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
self.__known_discord_servers[server_id] = server = DiscordServer(name, server_id, channels)
return server
- async def __iter_user_posts_from_url(self, scrape_item: ScrapeItem, url: URL) -> None:
+ async def __iter_user_posts_from_url(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL) -> None:
async for json_resp in self.__api_pager(url):
n_posts = 0
@@ -545,7 +544,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
if n_posts < MAX_OFFSET_PER_CALL:
break
- async def __api_pager(self, url: URL, step_size: int | None = None) -> AsyncGenerator[Any, None]:
+ async def __api_pager(self, url: AbsoluteHttpURL, step_size: int | None = None) -> AsyncGenerator[Any, None]:
"""Yields JSON responses from API calls, with configurable increments."""
current_step_size = step_size if step_size is not None else MAX_OFFSET_PER_CALL
init_offset = int(url.query.get("o") or 0)
@@ -599,7 +598,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
if not post.title or not post.user_name:
raise ScrapeError(422)
- files: list[URL] = []
+ files: list[AbsoluteHttpURL] = []
for selector in (_POST.VIDEOS, _POST.IMAGES, _POST.ATTACHMENTS):
for file in soup.select(selector):
files.append(self.parse_url(css.get_attr(file, "href")))
diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.py
index d80afaf29..330c62bde 100644
--- a/cyberdrop_dl/crawlers/eporner.py
+++ b/cyberdrop_dl/crawlers/eporner.py
@@ -221,8 +221,8 @@ def get_best_quality(soup: BeautifulSoup) -> tuple[str, str]:
def get_info_dict(soup: BeautifulSoup) -> dict:
- info_js_script = soup.select_one(_SELECTORS.DATE_JS)
- info_dict: dict = javascript.parse_json_to_dict(info_js_script.text, use_regex=False)
+ info_js_script = css.select_one_get_text(soup, _SELECTORS.DATE_JS)
+ info_dict: dict = javascript.parse_json_to_dict(info_js_script, use_regex=False)
javascript.clean_dict(info_dict)
return info_dict
diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.py
index 99d6a41af..98d25203a 100644
--- a/cyberdrop_dl/crawlers/gofile.py
+++ b/cyberdrop_dl/crawlers/gofile.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import http
import re
from datetime import UTC, datetime, timedelta
@@ -16,8 +17,6 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from collections.abc import Mapping
- from yarl import URL
-
WT_REGEX = re.compile(r'appdata\.wt\s=\s"([^"]+)"')
API_ENTRYPOINT = AbsoluteHttpURL("https://api.gofile.io")
@@ -97,9 +96,7 @@ class GoFileCrawler(Crawler):
self._website_token_date = datetime.now(UTC) - timedelta(days=7)
async def async_startup(self) -> None:
- get_website_token = error_handling_wrapper(self.get_website_token)
await self.get_account_token(API_ENTRYPOINT)
- await get_website_token(self, PRIMARY_URL)
async def fetch(self, scrape_item: ScrapeItem) -> None:
if "d" in scrape_item.url.parts:
@@ -147,11 +144,11 @@ class GoFileCrawler(Crawler):
async def handle_children(self, children: Mapping[str, Node], scrape_item: ScrapeItem) -> None:
"""Sends files to downloader and adds subfolder to scrape queue."""
- subfolders: list[URL] = []
- unavailable: list[URL] = []
- dangerous: list[URL] = []
+ subfolders: list[AbsoluteHttpURL] = []
+ unavailable: list[AbsoluteHttpURL] = []
+ dangerous: list[AbsoluteHttpURL] = []
- def get_website_url(node: Node) -> URL:
+ def get_website_url(node: Node) -> AbsoluteHttpURL:
if node["type"] == "folder":
return PRIMARY_URL / "d" / (node.get("code") or node["id"])
return scrape_item.url.with_fragment(file["id"])
@@ -194,16 +191,17 @@ class GoFileCrawler(Crawler):
subfolder = scrape_item.create_child(url)
self.manager.task_group.create_task(self.run(subfolder))
+ tasks = []
for url in unavailable:
- new_scrape_item = scrape_item.create_new(url)
- await self.raise_error(new_scrape_item, 403)
+ tasks.append(self.raise_error(url, 403))
for url in dangerous:
- new_scrape_item = scrape_item.create_new(url)
- await self.raise_error(new_scrape_item, "Dangerous File")
+ tasks.append(self.raise_error(url, "Dangerous File"))
+
+ await asyncio.gather(*tasks)
@error_handling_wrapper
- async def raise_error(self, scrape_item: ScrapeItem | URL, status: str | int, message: str | None = None) -> None:
+ async def raise_error(self, url: AbsoluteHttpURL, status: str | int, message: str | None = None) -> None:
raise ScrapeError(status, message)
@error_handling_wrapper
@@ -213,6 +211,7 @@ class GoFileCrawler(Crawler):
self.headers["Authorization"] = f"Bearer {self.api_key}"
cookies = {"accountToken": self.api_key}
self.update_cookies(cookies)
+ await self.get_website_token()
async def _get_new_api_key(self) -> str:
api_url = API_ENTRYPOINT / "accounts"
@@ -223,7 +222,7 @@ class GoFileCrawler(Crawler):
return json_resp["data"]["token"]
- async def get_website_token(self, _: ScrapeItem | URL | None = None, update: bool = False) -> None:
+ async def get_website_token(self, update: bool = False) -> None:
"""Creates an anon GoFile account to use."""
if datetime.now(UTC) - self._website_token_date < timedelta(seconds=120):
return
diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.py
index 182c14404..96754b6fe 100644
--- a/cyberdrop_dl/crawlers/imagebam.py
+++ b/cyberdrop_dl/crawlers/imagebam.py
@@ -22,13 +22,17 @@ PRIMARY_URL = AbsoluteHttpURL("https://www.imagebam.com/")
class ImageBamCrawler(Crawler):
- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Album": "/view/...", "Image": "/view/...", "Direct links": ""}
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Album": "/view/...",
+ "Image": "/view/...",
+ "Direct links": "",
+ }
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "imagebam"
FOLDER_DOMAIN: ClassVar[str] = "ImageBam"
async def async_startup(self) -> None:
- self.set_cookies()
+ self.update_cookies({"nsfw_inter": "1"})
async def fetch(self, scrape_item: ScrapeItem) -> None:
if is_cdn(scrape_item.url):
@@ -55,7 +59,7 @@ class ImageBamCrawler(Crawler):
for _, new_scrape_item in self.iter_children(scrape_item, soup, IMAGES_SELECTOR):
if not self.check_album_results(new_scrape_item.url, results):
- self.manager.task_group.create_task(self.image(new_scrape_item))
+ self.manager.task_group.create_task(self.image(new_scrape_item, soup))
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> None:
@@ -78,11 +82,6 @@ class ImageBamCrawler(Crawler):
custom_filename, _ = self.get_filename_and_ext(title)
await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
- def set_cookies(self) -> None:
- """Set cookies to bypass confirmation."""
- cookies = {"nsfw_inter": "1"}
- self.update_cookies(cookies)
-
def is_cdn(url: AbsoluteHttpURL) -> bool:
return "imagebam" in url.host.split(".") and "." in url.host.rstrip(".com")
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 6ae51dae7..1acb3bc2d 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import itertools
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
from aiolimiter import AsyncLimiter
from mediafire import MediaFireApi, api
@@ -23,7 +23,10 @@ PRIMARY_URL = AbsoluteHttpURL("https://www.mediafire.com/")
class MediaFireCrawler(Crawler):
- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"File": "/file/...", "Folder": "/folder/..."}
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "File": "/file/...",
+ "Folder": "/folder/...",
+ }
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "mediafire"
@@ -40,7 +43,7 @@ class MediaFireCrawler(Crawler):
async def folder(self, scrape_item: ScrapeItem) -> None:
folder_key = scrape_item.url.parts[2]
try:
- folder_details: dict[str, dict] = self.api.folder_get_info(folder_key=folder_key)
+ folder_details: dict[str, dict] = self.api.folder_get_info(folder_key=folder_key) # type: ignore
except api.MediaFireApiError as e:
raise MediaFireError(status=e.code, message=e.message) from None
@@ -49,7 +52,9 @@ class MediaFireCrawler(Crawler):
for chunk in itertools.count(1):
try:
- folder_contents: dict = self.api.folder_get_content(folder_key, "files", chunk=chunk, chunk_size=100)
+ folder_contents: dict[str, Any] = self.api.folder_get_content(
+ folder_key, "files", chunk=chunk, chunk_size=100
+ ) # type: ignore
except api.MediaFireApiError as e:
raise MediaFireError(status=e.code, message=e.message) from None
diff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.py
index f3061df99..dbec52c7f 100644
--- a/cyberdrop_dl/crawlers/rule34video.py
+++ b/cyberdrop_dl/crawlers/rule34video.py
@@ -115,8 +115,8 @@ class Rule34VideoCrawler(Crawler):
def get_video_info(soup: BeautifulSoup) -> VideoInfo:
title = css.select_one_get_text(soup, VIDEO_TITLE_SELECTOR)
- info_js_script = soup.select_one(JS_SELECTOR)
- info: dict[str, str | dict] = javascript.parse_json_to_dict(info_js_script.text)
+ info_js_script = css.select_one_get_text(soup, JS_SELECTOR)
+ info: dict[str, str | dict] = javascript.parse_json_to_dict(info_js_script)
info["title"] = title
javascript.clean_dict(info)
log_debug(info)
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.py
index 20eca16f4..1f5545109 100644
--- a/cyberdrop_dl/crawlers/xhamster.py
+++ b/cyberdrop_dl/crawlers/xhamster.py
@@ -58,7 +58,7 @@ class XhamsterCrawler(Crawler):
raise ValueError
@error_handling_wrapper
- async def profile(self, scrape_item: ScrapeItem, paths_to_scrape) -> None:
+ async def profile(self, scrape_item: ScrapeItem) -> None:
profile_type, username = scrape_item.url.parts[1:3]
title = self.create_title(f"{username} [{profile_type.removesuffix('s')}]")
scrape_item.setup_as_profile(title)
diff --git a/cyberdrop_dl/utils/javascript.py b/cyberdrop_dl/utils/javascript.py
index d40af7d96..b4e21c07f 100644
--- a/cyberdrop_dl/utils/javascript.py
+++ b/cyberdrop_dl/utils/javascript.py
@@ -35,7 +35,7 @@ def parse_js_vars(js_text: str, use_regex: bool = False) -> dict:
return data
-def parse_json_to_dict(js_text: str, use_regex: bool = False) -> dict[str, Any] | list[Any]:
+def parse_json_to_dict(js_text: str, use_regex: bool = False) -> Any:
json_str = js_text.replace("\t", "").replace("\n", "").strip()
json_str = replace_quotes(json_str)
if use_regex: