Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: general type errors (#1084)
NTFSvolume06bf602be563
diff --git a/cyberdrop_dl/crawlers/_kemono_base.py b/cyberdrop_dl/crawlers/_kemono_base.pyindex 505440e38..f7684d244 100644--- a/cyberdrop_dl/crawlers/_kemono_base.py+++ b/cyberdrop_dl/crawlers/_kemono_base.py@@ -24,7 +24,6 @@ if TYPE_CHECKING:from aiohttp_client_cache.response import AnyResponsefrom bs4 import BeautifulSoup- from yarl import URLfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem@@ -62,7 +61,7 @@ class UserURL(NamedTuple):post_id: str | None = None@staticmethod- def parse(url: URL) -> UserURL:+ def parse(url: AbsoluteHttpURL) -> UserURL:if (n_parts := len(url.parts)) > 3:post_id = url.parts[5] if n_parts > 5 else Nonereturn UserURL(url.parts[1], url.parts[3], post_id)@@ -74,7 +73,7 @@ class UserPostURL(UserURL):post_id: str@staticmethod- def parse(url: URL) -> UserPostURL:+ def parse(url: AbsoluteHttpURL) -> UserPostURL:result = UserURL.parse(url)assert result.post_id, "Individual posts must have a post_id"return cast("UserPostURL", result)@@ -87,7 +86,7 @@ class DiscordURL(NamedTuple):channel_id: str | None = None # Only present for individual channels URLs@staticmethod- def parse(url: URL) -> DiscordURL:+ def parse(url: AbsoluteHttpURL) -> DiscordURL:return DiscordURL(*url.parts[3:5])@@ -459,7 +458,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):"""~~~~~~~~ PRIVATE METHODS, should never be overriden ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""- async def __handle_post(self, scrape_item: ScrapeItem, post: Post):+ async def __handle_post(self, scrape_item: ScrapeItem, post: Post) -> None:# Process files if the post was generated from an API callfor file in post.all_files:file_url = self.__make_file_url(file)@@ -473,12 +472,12 @@ class KemonoBaseCrawler(Crawler, is_abc=True):self._handle_post_content(scrape_item, post)- def __make_file_url(self, file: File) -> URL:+ def __make_file_url(self, file: File) -> AbsoluteHttpURL:server = self.__known_attachment_servers.get(file["path"], "")url = server + f"/data{file['path']}"return self.parse_url(url).with_query(f=file["name"])- def __make_api_url_w_offset(self, path: str, og_url: URL) -> URL:+ def __make_api_url_w_offset(self, path: str, og_url: AbsoluteHttpURL) -> AbsoluteHttpURL:api_url = self.API_ENTRYPOINT / pathoffset = int(og_url.query.get("o", 0))if query := og_url.query.get("q"):@@ -519,7 +518,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):self.__known_discord_servers[server_id] = server = DiscordServer(name, server_id, channels)return server- async def __iter_user_posts_from_url(self, scrape_item: ScrapeItem, url: URL) -> None:+ async def __iter_user_posts_from_url(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL) -> None:async for json_resp in self.__api_pager(url):n_posts = 0@@ -545,7 +544,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):if n_posts < MAX_OFFSET_PER_CALL:break- async def __api_pager(self, url: URL, step_size: int | None = None) -> AsyncGenerator[Any, None]:+ async def __api_pager(self, url: AbsoluteHttpURL, step_size: int | None = None) -> AsyncGenerator[Any, None]:"""Yields JSON responses from API calls, with configurable increments."""current_step_size = step_size if step_size is not None else MAX_OFFSET_PER_CALLinit_offset = int(url.query.get("o") or 0)@@ -599,7 +598,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):if not post.title or not post.user_name:raise ScrapeError(422)- files: list[URL] = []+ files: list[AbsoluteHttpURL] = []for selector in (_POST.VIDEOS, _POST.IMAGES, _POST.ATTACHMENTS):for file in soup.select(selector):files.append(self.parse_url(css.get_attr(file, "href")))diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.pyindex d80afaf29..330c62bde 100644--- a/cyberdrop_dl/crawlers/eporner.py+++ b/cyberdrop_dl/crawlers/eporner.py@@ -221,8 +221,8 @@ def get_best_quality(soup: BeautifulSoup) -> tuple[str, str]:def get_info_dict(soup: BeautifulSoup) -> dict:- info_js_script = soup.select_one(_SELECTORS.DATE_JS)- info_dict: dict = javascript.parse_json_to_dict(info_js_script.text, use_regex=False)+ info_js_script = css.select_one_get_text(soup, _SELECTORS.DATE_JS)+ info_dict: dict = javascript.parse_json_to_dict(info_js_script, use_regex=False)javascript.clean_dict(info_dict)return info_dictdiff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.pyindex 99d6a41af..98d25203a 100644--- a/cyberdrop_dl/crawlers/gofile.py+++ b/cyberdrop_dl/crawlers/gofile.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport httpimport refrom datetime import UTC, datetime, timedelta@@ -16,8 +17,6 @@ from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from collections.abc import Mapping- from yarl import URL-WT_REGEX = re.compile(r'appdata\.wt\s=\s"([^"]+)"')API_ENTRYPOINT = AbsoluteHttpURL("https://api.gofile.io")@@ -97,9 +96,7 @@ class GoFileCrawler(Crawler):self._website_token_date = datetime.now(UTC) - timedelta(days=7)async def async_startup(self) -> None:- get_website_token = error_handling_wrapper(self.get_website_token)await self.get_account_token(API_ENTRYPOINT)- await get_website_token(self, PRIMARY_URL)async def fetch(self, scrape_item: ScrapeItem) -> None:if "d" in scrape_item.url.parts:@@ -147,11 +144,11 @@ class GoFileCrawler(Crawler):async def handle_children(self, children: Mapping[str, Node], scrape_item: ScrapeItem) -> None:"""Sends files to downloader and adds subfolder to scrape queue."""- subfolders: list[URL] = []- unavailable: list[URL] = []- dangerous: list[URL] = []+ subfolders: list[AbsoluteHttpURL] = []+ unavailable: list[AbsoluteHttpURL] = []+ dangerous: list[AbsoluteHttpURL] = []- def get_website_url(node: Node) -> URL:+ def get_website_url(node: Node) -> AbsoluteHttpURL:if node["type"] == "folder":return PRIMARY_URL / "d" / (node.get("code") or node["id"])return scrape_item.url.with_fragment(file["id"])@@ -194,16 +191,17 @@ class GoFileCrawler(Crawler):subfolder = scrape_item.create_child(url)self.manager.task_group.create_task(self.run(subfolder))+ tasks = []for url in unavailable:- new_scrape_item = scrape_item.create_new(url)- await self.raise_error(new_scrape_item, 403)+ tasks.append(self.raise_error(url, 403))for url in dangerous:- new_scrape_item = scrape_item.create_new(url)- await self.raise_error(new_scrape_item, "Dangerous File")+ tasks.append(self.raise_error(url, "Dangerous File"))++ await asyncio.gather(*tasks)@error_handling_wrapper- async def raise_error(self, scrape_item: ScrapeItem | URL, status: str | int, message: str | None = None) -> None:+ async def raise_error(self, url: AbsoluteHttpURL, status: str | int, message: str | None = None) -> None:raise ScrapeError(status, message)@error_handling_wrapper@@ -213,6 +211,7 @@ class GoFileCrawler(Crawler):self.headers["Authorization"] = f"Bearer {self.api_key}"cookies = {"accountToken": self.api_key}self.update_cookies(cookies)+ await self.get_website_token()async def _get_new_api_key(self) -> str:api_url = API_ENTRYPOINT / "accounts"@@ -223,7 +222,7 @@ class GoFileCrawler(Crawler):return json_resp["data"]["token"]- async def get_website_token(self, _: ScrapeItem | URL | None = None, update: bool = False) -> None:+ async def get_website_token(self, update: bool = False) -> None:"""Creates an anon GoFile account to use."""if datetime.now(UTC) - self._website_token_date < timedelta(seconds=120):returndiff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.pyindex 182c14404..96754b6fe 100644--- a/cyberdrop_dl/crawlers/imagebam.py+++ b/cyberdrop_dl/crawlers/imagebam.py@@ -22,13 +22,17 @@ PRIMARY_URL = AbsoluteHttpURL("https://www.imagebam.com/")class ImageBamCrawler(Crawler):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Album": "/view/...", "Image": "/view/...", "Direct links": ""}+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Album": "/view/...",+ "Image": "/view/...",+ "Direct links": "",+ }PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "imagebam"FOLDER_DOMAIN: ClassVar[str] = "ImageBam"async def async_startup(self) -> None:- self.set_cookies()+ self.update_cookies({"nsfw_inter": "1"})async def fetch(self, scrape_item: ScrapeItem) -> None:if is_cdn(scrape_item.url):@@ -55,7 +59,7 @@ class ImageBamCrawler(Crawler):for _, new_scrape_item in self.iter_children(scrape_item, soup, IMAGES_SELECTOR):if not self.check_album_results(new_scrape_item.url, results):- self.manager.task_group.create_task(self.image(new_scrape_item))+ self.manager.task_group.create_task(self.image(new_scrape_item, soup))@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> None:@@ -78,11 +82,6 @@ class ImageBamCrawler(Crawler):custom_filename, _ = self.get_filename_and_ext(title)await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)- def set_cookies(self) -> None:- """Set cookies to bypass confirmation."""- cookies = {"nsfw_inter": "1"}- self.update_cookies(cookies)-def is_cdn(url: AbsoluteHttpURL) -> bool:return "imagebam" in url.host.split(".") and "." in url.host.rstrip(".com")diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 6ae51dae7..1acb3bc2d 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -1,7 +1,7 @@from __future__ import annotationsimport itertools-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, Any, ClassVarfrom aiolimiter import AsyncLimiterfrom mediafire import MediaFireApi, api@@ -23,7 +23,10 @@ PRIMARY_URL = AbsoluteHttpURL("https://www.mediafire.com/")class MediaFireCrawler(Crawler):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"File": "/file/...", "Folder": "/folder/..."}+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "File": "/file/...",+ "Folder": "/folder/...",+ }PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "mediafire"@@ -40,7 +43,7 @@ class MediaFireCrawler(Crawler):async def folder(self, scrape_item: ScrapeItem) -> None:folder_key = scrape_item.url.parts[2]try:- folder_details: dict[str, dict] = self.api.folder_get_info(folder_key=folder_key)+ folder_details: dict[str, dict] = self.api.folder_get_info(folder_key=folder_key) # type: ignoreexcept api.MediaFireApiError as e:raise MediaFireError(status=e.code, message=e.message) from None@@ -49,7 +52,9 @@ class MediaFireCrawler(Crawler):for chunk in itertools.count(1):try:- folder_contents: dict = self.api.folder_get_content(folder_key, "files", chunk=chunk, chunk_size=100)+ folder_contents: dict[str, Any] = self.api.folder_get_content(+ folder_key, "files", chunk=chunk, chunk_size=100+ ) # type: ignoreexcept api.MediaFireApiError as e:raise MediaFireError(status=e.code, message=e.message) from Nonediff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.pyindex f3061df99..dbec52c7f 100644--- a/cyberdrop_dl/crawlers/rule34video.py+++ b/cyberdrop_dl/crawlers/rule34video.py@@ -115,8 +115,8 @@ class Rule34VideoCrawler(Crawler):def get_video_info(soup: BeautifulSoup) -> VideoInfo:title = css.select_one_get_text(soup, VIDEO_TITLE_SELECTOR)- info_js_script = soup.select_one(JS_SELECTOR)- info: dict[str, str | dict] = javascript.parse_json_to_dict(info_js_script.text)+ info_js_script = css.select_one_get_text(soup, JS_SELECTOR)+ info: dict[str, str | dict] = javascript.parse_json_to_dict(info_js_script)info["title"] = titlejavascript.clean_dict(info)log_debug(info)diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex 20eca16f4..1f5545109 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -58,7 +58,7 @@ class XhamsterCrawler(Crawler):raise ValueError@error_handling_wrapper- async def profile(self, scrape_item: ScrapeItem, paths_to_scrape) -> None:+ async def profile(self, scrape_item: ScrapeItem) -> None:profile_type, username = scrape_item.url.parts[1:3]title = self.create_title(f"{username} [{profile_type.removesuffix('s')}]")scrape_item.setup_as_profile(title)diff --git a/cyberdrop_dl/utils/javascript.py b/cyberdrop_dl/utils/javascript.pyindex d40af7d96..b4e21c07f 100644--- a/cyberdrop_dl/utils/javascript.py+++ b/cyberdrop_dl/utils/javascript.py@@ -35,7 +35,7 @@ def parse_js_vars(js_text: str, use_regex: bool = False) -> dict:return data-def parse_json_to_dict(js_text: str, use_regex: bool = False) -> dict[str, Any] | list[Any]:+def parse_json_to_dict(js_text: str, use_regex: bool = False) -> Any:json_str = js_text.replace("\t", "").replace("\n", "").strip()json_str = replace_quotes(json_str)if use_regex: