Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: type errors from ruff refactor (#1874)
NTFSvolume4f4a1fe462e7
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex 8159283af..3f2d3d89a 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -449,7 +449,9 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):for article in soup.select(self.SELECTORS.posts.article):current_post = ForumPost.new(article, self.SELECTORS.posts)continue_scraping, scrape_this_post = check_post_id(- thread.post_id, current_post.id, self.scrape_single_forum_post+ thread.post_id,+ current_post.id,+ scrape_single_forum_post=self.scrape_single_forum_post,)if scrape_this_post:post_url = self.make_post_url(thread, current_post.id)@@ -536,7 +538,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):@final@error_handling_wrapperasync def process_child(self, scrape_item: ScrapeItem, link_str: str, *, embeds: bool = False) -> None:- link_str_ = pre_process_child(link_str, embeds)+ link_str_ = pre_process_child(link_str, embeds=embeds)if not link_str_:return Nonelink = await self.get_absolute_link(link_str_)diff --git a/cyberdrop_dl/crawlers/_hls.py b/cyberdrop_dl/crawlers/_hls.pyindex fcc874483..97c36a169 100644--- a/cyberdrop_dl/crawlers/_hls.py+++ b/cyberdrop_dl/crawlers/_hls.py@@ -88,6 +88,6 @@ class SimpleHLSParser(HLSParser):_session: aiohttp.ClientSession@override- async def request_text(self, url: AbsoluteHttpURL, /, headers: Mapping[str, str] | None = None) -> str:+ async def request_text(self, url: AbsoluteHttpURL, /, *, headers: Mapping[str, str] | None = None) -> str:async with self._session.get(url, headers=headers) as resp:return await resp.text()diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex cff83a638..840e09fa7 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -175,6 +175,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):FOLDER_DOMAIN: ClassVar[str] = ""DOMAIN: ClassVar[str]PRIMARY_URL: ClassVar[AbsoluteHttpURL]+ _FORUM: ClassVar[bool] = False_RATE_LIMIT: ClassVar[RateLimit] = 25, 1_DOWNLOAD_SLOTS: ClassVar[int | None] = None@@ -598,7 +599,6 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):self,filename: str,*,- forum: bool = False,assume_ext: str | None = ".mp4",mime_type: str | None = None,) -> tuple[str, str]:@@ -607,10 +607,10 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):If that fails, appends `assume_ext` and tries again, but only if the user had exclude_files_with_no_extension = `False`"""try:- return get_filename_and_ext(filename, mime_type, xenforo=forum)+ return get_filename_and_ext(filename, mime_type, xenforo=self._FORUM)except NoExtensionError:if self.allow_no_extension and assume_ext:- return get_filename_and_ext(filename + assume_ext, mime_type, xenforo=forum)+ return get_filename_and_ext(filename + assume_ext, mime_type, xenforo=self._FORUM)raise@final@@ -799,12 +799,12 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):@final@classmethod- def parse_date(cls, date_or_datetime: str, /, format: str) -> dates.TimeStamp | None: # noqa: A002+ def parse_date(cls, date_or_datetime: str, /, format: str) -> dates.TimeStamp: # noqa: A002return dates.to_timestamp(dates.parse_format(date_or_datetime, format))@final@classmethod- def parse_iso_date(cls, date_or_datetime: str, /) -> dates.TimeStamp | None:+ def parse_iso_date(cls, date_or_datetime: str, /) -> dates.TimeStamp:return dates.to_timestamp(dates.parse_iso(date_or_datetime))async def _get_redirect_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:diff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.pyindex 716b50f39..cd9b6ddf0 100644--- a/cyberdrop_dl/crawlers/efukt.py+++ b/cyberdrop_dl/crawlers/efukt.py@@ -4,10 +4,8 @@ from pathlib import Pathfrom typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths-from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper-from cyberdrop_dl.utils.dates import to_timestamp+from cyberdrop_dl.utils import css, dates, error_handling_wrapperif TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem@@ -67,10 +65,8 @@ class EfuktCrawler(Crawler):soup = await self.request_soup(scrape_item.url)date_str = css.select_text(soup, _SELECTORS.DATE).split(" ", 1)[-1]- datetime = self._parse_date(date_str, "%m/%d/%y")- if not datetime:- raise ScrapeError(422)- scrape_item.uploaded_at = to_timestamp(datetime)+ date = dates.parse_format(date_str, "%m/%d/%y")+ scrape_item.uploaded_at = dates.to_timestamp(date)if is_image_or_gif(scrape_item.url):link = self.parse_url(css.select(soup, _SELECTORS.IMAGE, "src"))@@ -80,7 +76,7 @@ class EfuktCrawler(Crawler):item_id = scrape_item.url.query.get("id") or scrape_item.url.name.partition("_")[0]title = Path(css.select_text(soup, _SELECTORS.TITLE)).as_posix().replace("/", "-")filename, ext = self.get_filename_and_ext(link.name)- custom_filename = self.create_custom_filename(f"{datetime.date().isoformat()} {title}", ext, file_id=item_id)+ custom_filename = self.create_custom_filename(f"{date.date().isoformat()} {title}", ext, file_id=item_id)# Video links expire, but the path is always the same, only query params changeawait self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=link)diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.pyindex c4f9ba481..db38cef90 100644--- a/cyberdrop_dl/crawlers/kemono.py+++ b/cyberdrop_dl/crawlers/kemono.py@@ -8,7 +8,7 @@ from collections import defaultdictfrom collections.abc import Generatorfrom datetime import datetime # noqa: TC003from pathlib import Path-from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Concatenate, Literal, NamedTuple, ParamSpec+from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Concatenate, NamedTuple, ParamSpecfrom pydantic import BeforeValidator, Field@@ -321,7 +321,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):@fallback_if_no_api@error_handling_wrapper- async def favorites(self, scrape_item: ScrapeItem, type_: Literal["post", "artist"]) -> None:+ async def favorites(self, scrape_item: ScrapeItem, type_: str) -> None:if not self.session_cookie:msg = "No session cookie found in the config file, cannot scrape favorites"raise ScrapeError(401, msg)@@ -366,7 +366,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):if not server:continue- path = attach.get("path")+ path = attach.get("path", "")if previous_server := self.__known_attachment_servers.get(path):if previous_server != server:msg = (diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.pyindex c083c48d0..054a30948 100644--- a/cyberdrop_dl/crawlers/onedrive.py+++ b/cyberdrop_dl/crawlers/onedrive.py@@ -67,7 +67,7 @@ class OneDriveFile(OneDriveItem):download_url: AbsoluteHttpURL@classmethod- def from_api_response(cls, json_resp: dict, access_details: AccessDetails) -> Self:+ def from_api_response(cls, json_resp: dict[str, Any], access_details: AccessDetails) -> Self:info = parse_api_response(json_resp, access_details)download_url_str = json_resp["@content.downloadUrl"]info["download_url"] = AbsoluteHttpURL(download_url_str, encoded="%" in download_url_str)@@ -79,7 +79,7 @@ class OneDriveFolder(OneDriveItem):children: list[dict[str, Any]]@classmethod- def from_api_response(cls, json_resp: dict, access_details: AccessDetails) -> Self:+ def from_api_response(cls, json_resp: dict[str, Any], access_details: AccessDetails) -> Self:info = parse_api_response(json_resp, access_details)info["children"] = json_resp["children"]return cls(**info)@@ -147,7 +147,7 @@ class OneDriveCrawler(Crawler):raise ScrapeError(401)api_url = create_api_url(access_details)- json_resp: dict = await self.make_api_request(api_url)+ json_resp: dict[str, Any] = await self.make_api_request(api_url)if not is_folder(json_resp):file = OneDriveFile.from_api_response(json_resp, access_details)@@ -223,7 +223,7 @@ def is_folder(json_resp: dict[str, Any]) -> bool:return bool(json_resp.get("folder"))-def parse_api_response(json_resp: dict, access_details: AccessDetails) -> dict[str, Any]:+def parse_api_response(json_resp: dict[str, Any], access_details: AccessDetails) -> dict[str, Any]:web_url_str: str = json_resp["webUrl"]item_id = json_resp["id"]date_str = json_resp["fileSystemInfo"]["lastModifiedDateTime"]diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.pyindex dbf672b6c..41402c6bc 100644--- a/cyberdrop_dl/crawlers/pornhub.py+++ b/cyberdrop_dl/crawlers/pornhub.py@@ -81,7 +81,7 @@ class Media(TypedDict):width: intformat: Literal["hls", "mp4"]videoUrl: str- quality: str | list+ quality: str | list[str]class Format(NamedTuple):diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.pyindex 0bb896253..0d1b4be7f 100644--- a/cyberdrop_dl/crawlers/rumble.py+++ b/cyberdrop_dl/crawlers/rumble.py@@ -158,7 +158,7 @@ class RumbleCrawler(Crawler):_, ext = self.get_filename_and_ext(best_format.url.name)video_name = self.create_custom_filename(video.title, ext, file_id=embed_id, resolution=best_format.resolution)- scrape_item.uploaded_at = self.parse_iso_date(video["uploadDate"])+ scrape_item.uploaded_at = self.parse_iso_date(video.upload_date)scrape_item.url = video.urlself.create_task(self.handle_file(@@ -203,6 +203,6 @@ class RumbleCrawler(Crawler):)if hls_formats:- hls_formats = await aio.map(resolve_m3u8, hls_formats)+ hls_formats = await aio.map(resolve_m3u8, hls_formats, task_limit=10)return max((*hls_formats, *other_formats))diff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.pyindex c46bbf0f7..7ed3505e9 100644--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py@@ -1,12 +1,15 @@from __future__ import annotations-from typing import ClassVar+from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import error_handling_wrapperfrom .xenforo import XenforoCrawler+if TYPE_CHECKING:+ import yarl+_confirmation_data = ({"xhr": "1", "download": "1"},)@@ -30,8 +33,15 @@ class F95ZoneCrawler(XenforoCrawler):return url.with_path(url.path.replace("/thumb/", ""))@classmethod- def parse_url(cls, link: str) -> AbsoluteHttpURL:- url = super().parse_url(link)+ def parse_url(+ cls,+ url: yarl.URL | str,+ /,+ relative_to: AbsoluteHttpURL | None = None,+ *,+ trim: bool | None = None,+ ) -> AbsoluteHttpURL:+ url = super().parse_url(url, relative_to, trim=trim)if cls.is_thumbnail(url):return cls.thumbnail_to_img(url)return urldiff --git a/cyberdrop_dl/crawlers/xenforo/titsintops.py b/cyberdrop_dl/crawlers/xenforo/titsintops.pyindex 61eeceb97..182a99944 100644--- a/cyberdrop_dl/crawlers/xenforo/titsintops.py+++ b/cyberdrop_dl/crawlers/xenforo/titsintops.py@@ -8,6 +8,7 @@ from cyberdrop_dl.utils import cssfrom .xenforo import XenforoCrawlerif TYPE_CHECKING:+ import yarlfrom bs4 import Tag@@ -17,8 +18,15 @@ class TitsInTopsCrawler(XenforoCrawler):FOLDER_DOMAIN: ClassVar[str] = "TitsInTops"@classmethod- def parse_url(cls, link: str) -> AbsoluteHttpURL:- return super().parse_url(_query_to_path(link))+ def parse_url(+ cls,+ url: yarl.URL | str,+ /,+ relative_to: AbsoluteHttpURL | None = None,+ *,+ trim: bool | None = None,+ ) -> AbsoluteHttpURL:+ return super().parse_url(_query_to_path(str(url)), relative_to, trim=trim)def is_username_or_attachment(self, link_obj: Tag) -> bool:text = css.text(link_obj)diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.pyindex 861f4e8d0..42155fcd7 100644--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py@@ -74,15 +74,11 @@ class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):PAGE_URL_PART_NAME: ClassVar[str] = "page"IGNORE_EMBEDED_IMAGES_SRC: ClassVar[bool] = TrueLOGIN_USER_COOKIE_NAME: ClassVar[str] = "xf_user"+ _FORUM: ClassVar[bool] = True# Attachments hosts should technically be defined on each specific Crawler, but they do no harm hereATTACHMENT_HOSTS = "smgmedia", "attachments.f95zone"login_required = True- def get_filename_and_ext(self, filename: str) -> tuple[str, str]:- # The `forum` keyword is misleading now. It only works for Xenforo sites, not every forum- # TODO: Change `forum` parameter to `xenforo`- return super().get_filename_and_ext(filename, forum=True)-@error_handling_wrapperasync def xf_login(self, login_url: AbsoluteHttpURL, session_cookie: str, username: str, password: str) -> None:"""Logic to login as a Xenforo userdiff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex b70c84a58..e1c9fe16d 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -190,7 +190,7 @@ class XhamsterCrawler(Crawler):initials = await self._get_window_initials(next_page_url)images = initials["photosGalleryModel"]["photos"]- def _handle_img(self, scrape_item: ScrapeItem, img: dict[str, Any], results: dict[str, int]) -> None:+ def _handle_img(self, scrape_item: ScrapeItem, img: dict[str, Any], results: dict[str, bool]) -> None:src, page_url = self.parse_url(img["imageURL"]), self.parse_url(img["pageURL"])if self.check_album_results(src, results):returndiff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 619964b3c..9bee3dcc1 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -53,7 +53,7 @@ def select_text(tag: Tag, selector: str, *, strip: bool = True, decompose: str |if decompose:for trash in inner_tag.select(decompose):trash.decompose()- return text(inner_tag, strip)+ return text(inner_tag, strip=strip)def attr_or_none(tag: Tag, attribute: str) -> str | None: