Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: `StopIteration` error when reaching the last page (Kemono) (#1003)
NTFSvolume1c0f303ee3b4
diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.pyindex 4387eb7e5..21915c366 100644--- a/cyberdrop_dl/crawlers/kemono.py+++ b/cyberdrop_dl/crawlers/kemono.py@@ -7,10 +7,10 @@ import itertoolsimport refrom collections import defaultdictfrom datetime import datetime # noqa: TC003-from typing import TYPE_CHECKING, Annotated, Any, NamedTuple, NotRequired+from typing import TYPE_CHECKING, Annotated, Any, NamedTuple, NotRequired, castfrom pydantic import AliasChoices, BeforeValidator, Field-from typing_extensions import TypedDict # Compatible with python 3.11+from typing_extensions import TypedDict # Import from typing is not compatible with pydanticfrom yarl import URLfrom cyberdrop_dl.clients.errors import NoExtensionError, ScrapeError@@ -28,9 +28,6 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem-# TODO: Check what's the maximun offset we can use-# TODO: Verify if we need to use different offsets for each url type / service-# Right now is 50 becuase that's the max when scraping from the web page, but could be higher when using the APIMAX_OFFSET_PER_CALL = 50LINK_REGEX = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))")POST_SELECTOR = "article.post-card a"@@ -56,29 +53,41 @@ class PostSelectors:_POST = PostSelectors()-class URLInfo(NamedTuple):+class UserURL(NamedTuple):# format: https://kemono.su/<service>/user/<user_id>/post/<post_id># ex: https://kemono.su/patreon/user/92916478/post/87864845- service: str = "Unknown"- user_: str = "" # the literal word "user"- user_id: str = "Unknown"- post_: str = "" # the literal word "post"- post_id: str | None = None # Only present in individual posts URLs+ service: str+ user_id: str+ post_id: str | None = None++ @staticmethod+ def parse(url: URL) -> UserURL:+ if n_parts := len(url.parts) > 3:+ post_id = url.parts[5] if n_parts > 5 else None+ return UserURL(url.parts[1], url.parts[3], post_id)+ msg = "Invalid user URL"+ raise ValueError(msg)+++class UserPostURL(UserURL):+ post_id: str@staticmethod- def parse(url: URL) -> URLInfo:- return URLInfo(*url.parts[1:6])+ def parse(url: URL) -> UserPostURL:+ result = UserURL.parse(url)+ assert result.post_id, "Individual posts must have a post_id"+ return cast("UserPostURL", result)-class DiscordURLInfo(NamedTuple):+class DiscordURL(NamedTuple):# format: https://kemono.su/discord/server/<server_id>/<channel_id># ex: https://kemono.su/discord/server/891670433978531850/892624523034255371server_id: strchannel_id: str | None = None # Only present for individual channels URLs@staticmethod- def parse(url: URL) -> DiscordURLInfo:- return DiscordURLInfo(*url.parts[3:5])+ def parse(url: URL) -> DiscordURL:+ return DiscordURL(*url.parts[3:5])class DiscordChannel(NamedTuple):@@ -136,7 +145,7 @@ class UserPost(Post):return User(self.service, self.user_id)@property- def web_path_qs(self):+ def web_path_qs(self) -> str:return f"{self.service}/user/{self.user_id}/post/{self.id}"@@ -145,7 +154,7 @@ class DiscordPost(Post):channel_id: str = Field(validation_alias="channel")@property- def web_path_qs(self):+ def web_path_qs(self) -> str:return f"discord/server/{self.server_id}/{self.channel_id}#{self.id}"@@ -177,7 +186,7 @@ def fallback_if_no_api(func: Callable[..., Coroutine[None, None, Any]]) -> Calla"""Calls a fallback method is the current instance does not define an API"""@functools.wraps(func)- async def wrapper(self: KemonoCrawler, *args, **kwargs):+ async def wrapper(self: KemonoCrawler, *args, **kwargs) -> Any:if self.api_entrypoint:return await func(self, *args, **kwargs)fallback_func = getattr(self, f"{func.__name__}_w_no_api", None)@@ -213,7 +222,7 @@ class KemonoCrawler(Crawler):self.session_cookie = self.manager.config_manager.authentication_data.kemono.sessionasync def async_startup(self) -> None:- def check_kemono_page(response: AnyResponse):+ def check_kemono_page(response: AnyResponse) -> bool:if any(x in response.url.parts for x in self.services):return Falseif "discord/channel" in response.url.path:@@ -265,29 +274,35 @@ class KemonoCrawler(Crawler):@error_handling_wrapperasync def profile(self, scrape_item: ScrapeItem) -> None:"""Scrapes a profile."""- url_info = URLInfo.parse(scrape_item.url)- api_url = self.__make_api_url_w_offset(f"{url_info.service}/user/{url_info.user_id}", scrape_item.url)- scrape_item.setup_as_profile("") # Title will be added by self._handle_user_post+ url_info = UserURL.parse(scrape_item.url)+ path = f"{url_info.service}/user/{url_info.user_id}"+ api_url = self.__make_api_url_w_offset(path, scrape_item.url)+ scrape_item.setup_as_profile("")await self.__iter_user_posts_from_url(scrape_item, api_url)@fallback_if_no_api@error_handling_wrapperasync def discord(self, scrape_item: ScrapeItem) -> None:"""Scrapes a discord server or channel."""- url_info = DiscordURLInfo.parse(scrape_item.url)- if not url_info.channel_id: # Download all channels- scrape_item.setup_as_forum("") # Title will be added by self._handle_discord_post- server = await self.__get_discord_server(url_info.server_id)- for channel in server.channels:- url = self.primary_base_domain / "discord/server" / url_info.server_id / channel.id- new_scrape_item = scrape_item.create_child(url)- self.manager.task_group.create_task(self.run(new_scrape_item))- return+ discord = DiscordURL.parse(scrape_item.url)+ if discord.channel_id:+ return await self.discord_channel(scrape_item, discord.channel_id)- api_url = self.__make_api_url_w_offset(f"discord/channel/{url_info.channel_id}", scrape_item.url)+ await self.discord_server(scrape_item, discord.server_id)++ async def discord_server(self, scrape_item: ScrapeItem, server_id: str) -> None:+ scrape_item.setup_as_forum("")+ server = await self.__get_discord_server(server_id)+ for channel in server.channels:+ url = self.primary_base_domain / "discord/server" / server_id / channel.id+ new_scrape_item = scrape_item.create_child(url)+ self.manager.task_group.create_task(self.run(new_scrape_item))+ scrape_item.add_children()++ async def discord_channel(self, scrape_item: ScrapeItem, channel_id: str) -> None:scrape_item.setup_as_profile("")- page_generator = self.__api_pager(api_url)- async for json_resp in page_generator:+ api_url = self.__make_api_url_w_offset(f"discord/channel/{channel_id}", scrape_item.url)+ async for json_resp in self.__api_pager(api_url):n_posts = 0for post in (DiscordPost(**entry) for entry in json_resp): # type: ignoren_posts += 1@@ -295,14 +310,15 @@ class KemonoCrawler(Crawler):new_scrape_item = scrape_item.create_child(link)await self._handle_discord_post(new_scrape_item, post)scrape_item.add_children()- await page_generator.asend(n_posts)++ if not n_posts < MAX_OFFSET_PER_CALL:+ break@fallback_if_no_api@error_handling_wrapperasync def post(self, scrape_item: ScrapeItem) -> None:"""Scrapes an user post."""- url_info = URLInfo.parse(scrape_item.url)- assert url_info.post_id, "Individual posts must have a post_id"+ url_info = UserPostURL.parse(scrape_item.url)path = f"{url_info.service}/user/{url_info.user_id}/post/{url_info.post_id}"api_url = self.__make_api_url_w_offset(path, scrape_item.url)async with self.request_limiter:@@ -489,9 +505,8 @@ class KemonoCrawler(Crawler):self.__known_discord_servers[server_id] = server = DiscordServer(name, server_id, channels)return server- async def __iter_user_posts_from_url(self, scrape_item: ScrapeItem, url: URL):- page_generator = self.__api_pager(url)- async for json_resp in page_generator:+ async def __iter_user_posts_from_url(self, scrape_item: ScrapeItem, url: URL) -> None:+ async for json_resp in self.__api_pager(url):n_posts = 0# From search results@@ -513,27 +528,24 @@ class KemonoCrawler(Crawler):await self._handle_user_post(new_scrape_item, post)scrape_item.add_children()- await page_generator.asend(n_posts)-- async def __api_pager(self, url: URL) -> AsyncGenerator[dict[str, Any], int]:- """Yields jsons response from api calls, with increments of `MAX_OFFSET_PER_CALL`+ if not n_posts < MAX_OFFSET_PER_CALL:+ break- Receive number of proccesed posts (n_posts) from the response. Stops when `n_posts` < MAX_OFFSET_PER_CALL`"""+ async def __api_pager(self, url: URL) -> AsyncGenerator[dict[str, Any]]:+ """Yields jsons response from api calls, with increments of `MAX_OFFSET_PER_CALL`"""init_offset = int(url.query.get("o") or 0)for offset in itertools.count(init_offset, MAX_OFFSET_PER_CALL):api_url = url.update_query(o=offset)async with self.request_limiter:json_resp: dict = await self.client.get_json(self.domain, api_url)- n_post = yield json_resp- if not n_post < MAX_OFFSET_PER_CALL:- break+ yield json_resp# ~~~~~~~~~~ NO API METHODS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~- async def discord_w_no_api(self, scrape_item: ScrapeItem):+ async def discord_w_no_api(self, scrape_item: ScrapeItem) -> Any:raise NotImplementedError- async def search_w_no_api(self, scrape_item: ScrapeItem):+ async def search_w_no_api(self, scrape_item: ScrapeItem) -> Any:raise NotImplementedError@error_handling_wrapper@@ -542,8 +554,9 @@ class KemonoCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)- url_info = URLInfo.parse(scrape_item.url)- api_url = self.primary_base_domain / url_info.service / "user" / url_info.user_id+ url_info = UserURL.parse(scrape_item.url)+ path = f"{url_info.service}/user/{url_info.user_id}"+ api_url = self.primary_base_domain / pathscrape_item.setup_as_profile("")init_offset = int(scrape_item.url.query.get("o") or 0)for offset in itertools.count(init_offset, MAX_OFFSET_PER_CALL):@@ -566,13 +579,12 @@ class KemonoCrawler(Crawler):async def post_w_no_api(self, scrape_item: ScrapeItem) -> None:"""Scrapes an user post."""- url_info = URLInfo.parse(scrape_item.url)- assert url_info.post_id, "Individual posts must have a post_id"+ url_info = UserPostURL.parse(scrape_item.url)async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)- post_info = PartialUserPost.from_soup(soup)- if not post_info.title or not post_info.user_name:+ post = PartialUserPost.from_soup(soup)+ if not post.title or not post.user_name:raise ScrapeError(422)files: list[URL] = []@@ -580,15 +592,15 @@ class KemonoCrawler(Crawler):for file in soup.select(selector):files.append(self.parse_url(file["href"])) # type: ignore- post = UserPost(+ full_post = UserPost(user_id=url_info.user_id,service=url_info.service,id=url_info.post_id,- title=post_info.title,- content=post_info.content,- published_or_added=post_info.date, # type: ignore+ title=post.title,+ content=post.content,+ published_or_added=post.date, # type: ignoresoup_attachments=files,)- await self._register_user_name(post.user, post_info.user_name)- await self._handle_user_post(scrape_item, post)+ await self._register_user_name(full_post.user, post.user_name)+ await self._handle_user_post(scrape_item, full_post)