Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: skip duplicate URLs (Kemono) (#1220)
NTFSvolume61515883213d
diff --git a/cyberdrop_dl/crawlers/_kemono_base.py b/cyberdrop_dl/crawlers/_kemono_base.pydeleted file mode 100644index 572c716fa..000000000--- a/cyberdrop_dl/crawlers/_kemono_base.py+++ /dev/null@@ -1,596 +0,0 @@-from __future__ import annotations--import asyncio-import functools-import itertools-import re-from collections import defaultdict-from datetime import datetime # noqa: TC003-from json import loads as json_loads-from typing import TYPE_CHECKING, Annotated, Any, ClassVar, NamedTuple, NotRequired, cast--from pydantic import AliasChoices, BeforeValidator, Field-from typing_extensions import TypedDict # Import from typing is not compatible with pydantic--from cyberdrop_dl.crawlers.crawler import Crawler-from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError-from cyberdrop_dl.models import AliasModel-from cyberdrop_dl.models.validators import falsy_as_none-from cyberdrop_dl.utils import css-from cyberdrop_dl.utils.dates import to_timestamp-from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts--if TYPE_CHECKING:- from collections.abc import AsyncGenerator, Callable, Coroutine, Generator-- from aiohttp_client_cache.response import AnyResponse- from bs4 import BeautifulSoup-- from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem---MAX_OFFSET_PER_CALL = 50-DISCORD_CHANNEL_PAGE_SIZE = 150-LINK_REGEX = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))")-POST_SELECTOR = "article.post-card a"---class PostSelectors:- FILES = "div[class*=__files]"- CONTENT = "div[class*=__content]"- ATTACHMENTS = "a[class*=__attachment-link]"- IMAGES = "div[class=fileThumb]"- VIDEOS = "video[class*=__video] source"- ALL_CONTENT = f"{FILES}, {CONTENT}"-- DATE_PUBLISHED = "div[class*=__published] time[class=timestamp]"- DATE_ADDED = "div[class*=__added]"- DATE = f"{DATE_PUBLISHED}, {DATE_ADDED}"-- TITLE = "h1[class*=__title]"- USERNAME = "a[class*=__user-name]"---_POST = PostSelectors()---class UserURL(NamedTuple):- # format: https://kemono.su/<service>/user/<user_id>/post/<post_id>- # ex: https://kemono.su/patreon/user/92916478/post/87864845- service: str- user_id: str- post_id: str | None = None-- @staticmethod- def parse(url: AbsoluteHttpURL) -> UserURL:- if (n_parts := len(url.parts)) > 3:- post_id = url.parts[5] if n_parts > 5 else None- return UserURL(url.parts[1], url.parts[3], post_id)- msg = "Invalid user URL"- raise ValueError(msg)---class UserPostURL(UserURL):- post_id: str-- @staticmethod- def parse(url: AbsoluteHttpURL) -> UserPostURL:- result = UserURL.parse(url)- assert result.post_id, "Individual posts must have a post_id"- return cast("UserPostURL", result)---class DiscordURL(NamedTuple):- # format: https://kemono.su/discord/server/<server_id>/<channel_id>- # ex: https://kemono.su/discord/server/891670433978531850/892624523034255371- server_id: str- channel_id: str | None = None # Only present for individual channels URLs-- @staticmethod- def parse(url: AbsoluteHttpURL) -> DiscordURL:- return DiscordURL(*url.parts[3:5])---class DiscordChannel(NamedTuple):- name: str- id: str---class DiscordServer(NamedTuple):- name: str- id: str- channels: tuple[DiscordChannel, ...]---class User(NamedTuple):- service: str- id: str---class File(TypedDict):- name: NotRequired[str] # Sometimes present- path: str- server: NotRequired[str] # Sometimes present in attachments---FileOrNone = Annotated[File | None, BeforeValidator(falsy_as_none)]---class Post(AliasModel):- id: str- content: str = ""- file: FileOrNone = None- attachments: list[File] = [] # noqa: RUF012- published_or_added: datetime | None = Field(None, validation_alias=AliasChoices("published", "added"))- soup_attachments: list[Any] = [] # noqa: RUF012, `Any` to skip validation, but these are `yarl.URL`. We generate them internally so no validation is needed- date: int | None = None-- def model_post_init(self, *_) -> None:- if self.published_or_added:- self.date = to_timestamp(self.published_or_added)-- @property- def all_files(self) -> Generator[File]:- if self.file:- yield self.file- yield from self.attachments---class UserPost(Post):- service: str- user_id: str = Field(validation_alias="user")- title: str-- @property- def user(self) -> User:- return User(self.service, self.user_id)-- @property- def web_path_qs(self) -> str:- return f"{self.service}/user/{self.user_id}/post/{self.id}"---class DiscordPost(Post):- server_id: str = Field(validation_alias="server")- channel_id: str = Field(validation_alias="channel")-- @property- def web_path_qs(self) -> str:- return f"discord/server/{self.server_id}/{self.channel_id}#{self.id}"---class PartialUserPost(NamedTuple):- """A simplified version of Post that we can built from a soup, for sites that do not have an API-- Pros: We can get the post data + user_name in a single request-- Cons: We need to make a request for every individual post"""-- title: str = ""- content: str = ""- user_name: str = ""- date: str | None = None-- @staticmethod- def from_soup(soup: BeautifulSoup) -> PartialUserPost:- info = {}- names = "title", "content", "user_name", "date"- selectors = (_POST.TITLE, _POST.ALL_CONTENT, _POST.USERNAME, _POST.DATE)- for name, selector in zip(names, selectors, strict=True):- if tag := soup.select_one(selector):- info[name] = tag.text.strip()-- return PartialUserPost(**info)---def fallback_if_no_api(func: Callable[..., Coroutine[None, None, Any]]) -> Callable[..., Coroutine[None, None, Any]]:- """Calls a fallback method is the current instance does not define an API"""-- @functools.wraps(func)- async def wrapper(self: KemonoBaseCrawler, *args, **kwargs) -> Any:- if hasattr(self, "API_ENTRYPOINT"):- return await func(self, *args, **kwargs)- fallback_func = getattr(self, f"{func.__name__}_w_no_api", None)- if not fallback_func:- raise ValueError- return await fallback_func(self, *args, **kwargs)-- return wrapper---class KemonoBaseCrawler(Crawler, is_abc=True):- SUPPORTED_PATHS: ClassVar[dict[str, str]] = {- "Model": "/<service>/user/<user>",- "Favorites": "/favorites/<user>",- "Search": "/search?...",- "Individual Post": "<service>/<user>/post/...",- "Direct links": "",- }- DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {title}"- API_ENTRYPOINT: ClassVar[AbsoluteHttpURL]- SERVICES: ClassVar[tuple[str, ...]] = ()-- def __init_subclass__(cls, **kwargs) -> None:- super().__init_subclass__(**kwargs)- REQUIRED_FIELDS = ("SERVICES",)- for field_name in REQUIRED_FIELDS:- assert getattr(cls, field_name, None), f"Subclass {cls.__name__} must override: {field_name}"-- def __post_init__(self) -> None:- self._user_names: dict[User, str] = {}- self.__known_discord_servers: dict[str, DiscordServer] = {}- self.__known_attachment_servers: dict[str, str] = {}- self.__discord_servers_locks: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)- self.session_cookie = self.manager.config_manager.authentication_data.kemono.session-- async def async_startup(self) -> None:- def check_kemono_page(response: AnyResponse) -> bool:- if any(x in response.url.parts for x in self.SERVICES):- return False- if "discord/channel" in response.url.path:- return False- return True-- self.register_cache_filter(self.PRIMARY_URL, check_kemono_page)- if getattr(self, "API_ENTRYPOINT", None):- await self.__get_usernames(self.API_ENTRYPOINT / "creators")-- async def fetch(self, scrape_item: ScrapeItem) -> None:- if "thumbnails" in scrape_item.url.parts:- return await self.handle_direct_link(scrape_item)- if "post" in scrape_item.url.parts:- return await self.post(scrape_item)- if scrape_item.url.name == "posts":- if not scrape_item.url.query.get("q"):- raise ValueError- return await self.search(scrape_item)- if any(x in scrape_item.url.parts for x in self.SERVICES):- return await self.profile(scrape_item)- if "favorites" in scrape_item.url.parts:- return await self.favorites(scrape_item)- await self.handle_direct_link(scrape_item)-- @fallback_if_no_api- @error_handling_wrapper- async def search(self, scrape_item: ScrapeItem) -> None:- query = scrape_item.url.query["q"]- api_url = self.__make_api_url_w_offset("posts", scrape_item.url)- title = self.create_title(f"{query} [search]")- scrape_item.setup_as_profile(title)- await self.__iter_user_posts_from_url(scrape_item, api_url)-- @fallback_if_no_api- @error_handling_wrapper- async def profile(self, scrape_item: ScrapeItem) -> None:- url_info = UserURL.parse(scrape_item.url)- path = f"{url_info.service}/user/{url_info.user_id}/posts"- api_url = self.__make_api_url_w_offset(path, scrape_item.url)- scrape_item.setup_as_profile("")- await self.__iter_user_posts_from_url(scrape_item, api_url)-- @fallback_if_no_api- @error_handling_wrapper- async def discord(self, scrape_item: ScrapeItem) -> None:- discord = DiscordURL.parse(scrape_item.url)- if discord.channel_id:- return await self.discord_channel(scrape_item, discord.channel_id)-- await self.discord_server(scrape_item, discord.server_id)-- async def discord_server(self, scrape_item: ScrapeItem, server_id: str) -> None:- scrape_item.setup_as_forum("")- server = await self.__get_discord_server(server_id)- for channel in server.channels:- url = self.PRIMARY_URL / "discord/server" / server_id / channel.id- new_scrape_item = scrape_item.create_child(url)- self.create_task(self.run(new_scrape_item))- scrape_item.add_children()-- async def discord_channel(self, scrape_item: ScrapeItem, channel_id: str) -> None:- scrape_item.setup_as_profile("")- api_url_template = self.__make_api_url_w_offset(f"discord/channel/{channel_id}", scrape_item.url)- async for json_response_list in self.__api_pager(api_url_template, step_size=DISCORD_CHANNEL_PAGE_SIZE):- if not isinstance(json_response_list, list):- error_msg = (- f"[{self.NAME}] Invalid API response for Discord channel '{channel_id}' posts (URL template: {api_url_template}). "- f"Expected a list, but got type {type(json_response_list).__name__}. "- f"Response data (truncated): {str(json_response_list)[:200]}"- )- raise ScrapeError(422, error_msg)- if not json_response_list:- break-- num_posts_in_page = 0- for post_data in json_response_list:- num_posts_in_page += 1- if not isinstance(post_data, dict):- error_msg = (- f"[{self.NAME}] Invalid post data type in list for Discord channel '{channel_id}' (URL template: {api_url_template}). "- f"Expected a dict for post data, but got type {type(post_data).__name__}. "- f"Post data (truncated): {str(post_data)[:200]}"- )- raise ScrapeError(422, error_msg)- post = DiscordPost.model_validate(post_data)- post_web_url = self.parse_url(post.web_path_qs)- new_scrape_item_for_post = scrape_item.create_child(post_web_url)- await self._handle_discord_post(new_scrape_item_for_post, post)- scrape_item.add_children()-- if num_posts_in_page < DISCORD_CHANNEL_PAGE_SIZE:- break-- @fallback_if_no_api- @error_handling_wrapper- async def post(self, scrape_item: ScrapeItem) -> None:- url_info = UserPostURL.parse(scrape_item.url)- path = f"{url_info.service}/user/{url_info.user_id}/post/{url_info.post_id}"- api_url = self.__make_api_url_w_offset(path, scrape_item.url)- json_resp: dict[str, Any] = await self.__api_request(api_url)- post = UserPost.model_validate(json_resp["post"])- self._register_attachments_servers(json_resp["attachments"])- await self._handle_user_post(scrape_item, post)-- def _register_attachments_servers(self, attachments: list[File]) -> None:- for attach in attachments:- if server := attach.get("server"):- path = attach["path"]- if previous_server := self.__known_attachment_servers.get(path):- if previous_server != server:- msg = f"[{self.NAME}] {path} found with multiple diferent servers: {server = } {previous_server = } "- self.log_debug(msg)- continue- self.__known_attachment_servers[path] = server-- @error_handling_wrapper- async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:- """Handles a direct link."""-- def clean_url(og_url: AbsoluteHttpURL) -> AbsoluteHttpURL:- if "thumbnails" in og_url.parts:- return remove_parts(og_url, "thumbnails")- return og_url-- scrape_item.url = clean_url(scrape_item.url)- link = clean_url(url or scrape_item.url)- try:- filename, ext = self.get_filename_and_ext(link.query.get("f") or link.name)- except NoExtensionError:- # Some patreon URLs have another URL as the filename: https://kemono.su/data/7a...27ad7e40bd.jpg?f=https://www.patreon.com/media-u/Z0F..00672794_- filename, ext = self.get_filename_and_ext(link.name)- await self.handle_file(link, scrape_item, filename, ext)-- @error_handling_wrapper- async def favorites(self, scrape_item: ScrapeItem) -> None:- if not self.session_cookie:- raise ScrapeError(401, "No session cookie found in the config file, cannot scrape favorites")-- is_post = scrape_item.url.query.get("type") == "post"- title = "My favorite posts" if is_post else "My favorite artists"- scrape_item.setup_as_profile(self.create_title(title))-- self.update_cookies({"session": self.session_cookie})- api_url = self.API_ENTRYPOINT / "account/favorites"- query_url = api_url.with_query(type="post" if is_post else "artist")- json_resp: list[dict] = await self.__api_request(query_url)- self.update_cookies({"session": ""})-- for item in json_resp:- if is_post:- post_id, user_id, service = item["id"], item["user"], item["service"]- url = self.PRIMARY_URL / service / "user" / user_id / "post" / post_id- else:- user_id, service = item["id"], item["service"]- url = self.PRIMARY_URL / service / "user" / user_id-- new_scrape_item = scrape_item.create_child(url)- self.create_task(self.run(new_scrape_item))-- # ~~~~~~~~ INTERNAL METHODS, not expected to be overriden, but could be ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-- async def _handle_user_post(self, scrape_item: ScrapeItem, post: UserPost):- user_name = self._user_names[post.user]- title = self.create_title(user_name, post.user_id)- scrape_item.setup_as_album(title, album_id=post.user_id)- scrape_item.possible_datetime = post.date- post_title = self.create_separate_post_title(post.title, post.id, post.date)- scrape_item.add_to_parent_title(post_title)-- await self.__handle_post(scrape_item, post)-- async def _handle_discord_post(self, scrape_item: ScrapeItem, post: DiscordPost):- server = await self.__get_discord_server(post.server_id)- title = self.create_title(f"{server.name} [discord]", server.id)- channel_name = next(c.name for c in server.channels if c.id == post.channel_id)- scrape_item.setup_as_album(title, album_id=server.id)- scrape_item.possible_datetime = post.date- scrape_item.add_to_parent_title(f"#{channel_name}")-- post_title = self.create_separate_post_title(None, post.id, post.date)- scrape_item.add_to_parent_title(post_title)-- await self.__handle_post(scrape_item, post)-- def _handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:- """Gets links out of content in post ans sends them to a new crawler."""- if not post.content:- return-- def gen_yarl_urls() -> Generator[AbsoluteHttpURL]:- for match in re.finditer(LINK_REGEX, post.content):- link = match.group().replace(".md.", ".")- try:- url = self.parse_url(link)- yield url- except Exception:- pass-- for link in gen_yarl_urls():- if not link.host or self.DOMAIN in link.host:- continue- new_scrape_item = scrape_item.create_child(link)- self.handle_external_links(new_scrape_item)- scrape_item.add_children()-- """~~~~~~~~ PRIVATE METHODS, should never be overriden ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""-- async def __handle_post(self, scrape_item: ScrapeItem, post: Post) -> None:- # Process files if the post was generated from an API call- for file in post.all_files:- file_url = self.__make_file_url(file)- await self.handle_direct_link(scrape_item, file_url)- scrape_item.add_children()-- # Process files if the post was generated from soup- for file_url in post.soup_attachments:- await self.handle_direct_link(scrape_item, file_url)- scrape_item.add_children()-- self._handle_post_content(scrape_item, post)-- def __make_file_url(self, file: File) -> AbsoluteHttpURL:- server = self.__known_attachment_servers.get(file["path"], "")- url = self.parse_url(server + f"/data{file['path']}")- return url.with_query(f=file.get("name") or url.name)-- def __make_api_url_w_offset(self, path: str, og_url: AbsoluteHttpURL) -> AbsoluteHttpURL:- api_url = self.API_ENTRYPOINT / path- offset = int(og_url.query.get("o", 0))- if query := og_url.query.get("q"):- return api_url.update_query(o=offset, q=query)- return api_url.update_query(o=offset)-- async def __get_usernames(self, api_url: AbsoluteHttpURL) -> None:- try:- json_resp: list[dict[str, Any]] = await self.__api_request(api_url)- self._user_names = {User(u["service"], u["id"]): u["name"] for u in json_resp}- except Exception:- pass- if not self._user_names:- self.log(f"Unable to get list of creators from {self.NAME}. Crawler has been disabled")- self.disabled = True-- async def __get_discord_server(self, server_id: str) -> DiscordServer:- """Get discord server information, making new API calls if needed."""- async with self.__discord_servers_locks[server_id]:- if server := self.__known_discord_servers.get(server_id):- return server-- api_url_server_profile = self.API_ENTRYPOINT / "discord" / "user" / server_id / "profile"- server_profile_json: dict[str, Any] = await self.__api_request(api_url_server_profile)- name = server_profile_json.get("name")- if not name:- name = f"Discord Server {server_id}"- api_url_channels = self.API_ENTRYPOINT / "discord/channel/lookup" / server_id- channels_json_resp: list[dict] = await self.__api_request(api_url_channels)- channels = tuple(DiscordChannel(channel["name"], channel["id"]) for channel in channels_json_resp)- self.__known_discord_servers[server_id] = server = DiscordServer(name, server_id, channels)- return server-- async def __iter_user_posts_from_url(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL) -> None:- async for json_resp in self.__api_pager(url):- n_posts = 0-- # From search results- if isinstance(json_resp, dict):- posts = json_resp.get("posts", [])- # From profile- elif isinstance(json_resp, list):- posts: list[dict[str, Any]] = json_resp- else:- raise ScrapeError(422)-- if not posts:- return-- for post in (UserPost.model_validate(entry) for entry in posts):- n_posts += 1- link = self.parse_url(post.web_path_qs)- new_scrape_item = scrape_item.create_child(link)- await self._handle_user_post(new_scrape_item, post)- scrape_item.add_children()-- if n_posts < MAX_OFFSET_PER_CALL:- break-- async def __api_pager(self, url: AbsoluteHttpURL, step_size: int | None = None) -> AsyncGenerator[Any, None]:- """Yields JSON responses from API calls, with configurable increments."""- current_step_size = step_size if step_size is not None else MAX_OFFSET_PER_CALL- init_offset = int(url.query.get("o") or 0)- for current_offset in itertools.count(init_offset, current_step_size):- api_url = url.update_query(o=current_offset)- yield await self.__api_request(api_url)-- # ~~~~~~~~~~ NO API METHODS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-- async def discord_w_no_api(self, scrape_item: ScrapeItem) -> Any:- raise NotImplementedError-- async def search_w_no_api(self, scrape_item: ScrapeItem) -> Any:- raise NotImplementedError-- @error_handling_wrapper- async def profile_w_no_api(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)-- url_info = UserURL.parse(scrape_item.url)- path = f"{url_info.service}/user/{url_info.user_id}"- api_url = self.PRIMARY_URL / path- scrape_item.setup_as_profile("")- init_offset = int(scrape_item.url.query.get("o") or 0)- for offset in itertools.count(init_offset, MAX_OFFSET_PER_CALL):- n_posts = 0- api_url = api_url.with_query(o=offset)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, api_url)-- for post in soup.select(POST_SELECTOR):- n_posts += 1- link = self.parse_url(css.get_attr(post, "href"))- new_scrape_item = scrape_item.create_child(link)- await self.post_w_no_api(new_scrape_item)- scrape_item.add_children()-- if n_posts < MAX_OFFSET_PER_CALL:- break-- @error_handling_wrapper- async def post_w_no_api(self, scrape_item: ScrapeItem) -> None:- url_info = UserPostURL.parse(scrape_item.url)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)-- post = PartialUserPost.from_soup(soup)- if not post.title or not post.user_name:- raise ScrapeError(422)-- files: list[AbsoluteHttpURL] = []- for selector in (_POST.VIDEOS, _POST.IMAGES, _POST.ATTACHMENTS):- for file in soup.select(selector):- files.append(self.parse_url(css.get_attr(file, "href")))-- full_post = UserPost(- user_id=url_info.user_id,- service=url_info.service,- id=url_info.post_id,- title=post.title,- content=post.content,- published_or_added=post.date, # type: ignore[reportArgumentType]- soup_attachments=files,- )-- self._user_names[full_post.user] = post.user_name- await self._handle_user_post(scrape_item, full_post)-- async def __api_request(self, url: AbsoluteHttpURL) -> Any:- """Get JSON from the API, with a custom Accept header."""-- async with self.request_limiter:- response, soup_or_none = await self.client._get(self.DOMAIN, url, headers={"Accept": "text/css"})-- if soup_or_none:- return json_loads(soup_or_none.text)-- # When using the 'text/css' header, the response is missing the charset header- # and charset detection may return a random codec if the body has non english chars, so we force utf-8- return json_loads(await response.text(encoding="utf-8"))diff --git a/cyberdrop_dl/crawlers/coomer.py b/cyberdrop_dl/crawlers/coomer.pyindex 29ce20c62..6594734f6 100644--- a/cyberdrop_dl/crawlers/coomer.py+++ b/cyberdrop_dl/crawlers/coomer.py@@ -2,11 +2,9 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar-from aiolimiter import AsyncLimiter-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from ._kemono_base import KemonoBaseCrawler, Post+from .kemono import KemonoBaseCrawler, Postif TYPE_CHECKING:from aiohttp_client_cache.response import AnyResponse@@ -21,10 +19,9 @@ class CoomerCrawler(KemonoBaseCrawler):SERVICES = "onlyfans", "fansly"OLD_DOMAINS: ClassVar[tuple[str, ...]] = "coomer.party", "coomer.su"- def __post_init__(self) -> None:- super().__post_init__()- self.request_limiter = AsyncLimiter(4, 1)- self.session_cookie = self.manager.config_manager.authentication_data.coomer.session+ @property+ def session_cookie(self) -> str:+ return self.manager.config_manager.authentication_data.coomer.sessionasync def async_startup(self) -> None:await super().async_startup()diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 2276568c0..31ee0b25c 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -150,6 +150,12 @@ class Crawler(ABC):def __post_init__(self) -> None: ... # noqa: B027+ @final+ @staticmethod+ def _assert_fields_overrides(subclass: type[Crawler], *fields: str):+ for field_name in fields:+ assert getattr(subclass, field_name, None), f"Subclass {subclass.__name__} must override: {field_name}"+def __init_subclass__(cls, is_abc: bool = False, is_generic: bool = False, generic_name: str = "", **kwargs) -> None:@@ -177,9 +183,7 @@ class Crawler(ABC):returnif not (cls.IS_FALLBACK_GENERIC or cls.IS_REAL_DEBRID):- REQUIRED_FIELDS = "PRIMARY_URL", "DOMAIN", "SUPPORTED_PATHS"- for field_name in REQUIRED_FIELDS:- assert getattr(cls, field_name, None), f"Subclass {cls.__name__} must override: {field_name}"+ Crawler._assert_fields_overrides(cls, "PRIMARY_URL", "DOMAIN", "SUPPORTED_PATHS")if cls.OLD_DOMAINS:cls.REPLACE_OLD_DOMAINS_REGEX = re.compile("|".join(cls.OLD_DOMAINS))diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.pyindex c49a2eb04..de5e01d51 100644--- a/cyberdrop_dl/crawlers/kemono.py+++ b/cyberdrop_dl/crawlers/kemono.py@@ -1,16 +1,534 @@from __future__ import annotations-from typing import ClassVar+import asyncio+import functools+import itertools+import re+from collections import defaultdict+from datetime import datetime # noqa: TC003+from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Concatenate, Literal, NamedTuple, NotRequired, ParamSpec-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem+from pydantic import AliasChoices, BeforeValidator, Field+from typing_extensions import TypedDict # Import from typing is not compatible with pydantic-from ._kemono_base import KemonoBaseCrawler+from cyberdrop_dl.crawlers.crawler import Crawler, auto_task_id+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError+from cyberdrop_dl.models import AliasModel+from cyberdrop_dl.models.validators import falsy_as_none+from cyberdrop_dl.utils import css+from cyberdrop_dl.utils.dates import to_timestamp+from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator, Callable, Coroutine, Generator++ from aiohttp_client_cache.response import AnyResponse+ from bs4 import BeautifulSoup++ from cyberdrop_dl.data_structures.url_objects import ScrapeItem++ _P = ParamSpec("_P")+++_DEFAULT_PAGE_SIZE = 50+_DISCORD_CHANNEL_PAGE_SIZE = 150+_POST_SELECTOR = "article.post-card a"+_find_http_urls = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))").finditer+++class PostSelectors:+ FILES = "div[class*=__files]"+ CONTENT = "div[class*=__content]"+ ATTACHMENTS = "a[class*=__attachment-link]"+ IMAGES = "div[class=fileThumb]"+ VIDEOS = "video[class*=__video] source"+ ALL_CONTENT = f"{FILES}, {CONTENT}"++ DATE_PUBLISHED = "div[class*=__published] time[class=timestamp]"+ DATE_ADDED = "div[class*=__added]"+ DATE = f"{DATE_PUBLISHED}, {DATE_ADDED}"++ TITLE = "h1[class*=__title]"+ USERNAME = "a[class*=__user-name]"+++class DiscordChannel(NamedTuple):+ name: str+ id: str+++class DiscordServer(NamedTuple):+ name: str+ id: str+ channels: tuple[DiscordChannel, ...]+++class User(NamedTuple):+ service: str+ id: str+++class File(TypedDict):+ name: NotRequired[str] # Sometimes present+ path: str+ server: NotRequired[str] # Sometimes present in attachments+++FileOrNone = Annotated[File | None, BeforeValidator(falsy_as_none)]+++class Post(AliasModel):+ id: str+ content: str = ""+ file: FileOrNone = None+ attachments: list[File] = [] # noqa: RUF012+ published_or_added: datetime | None = Field(None, validation_alias=AliasChoices("published", "added"))+ date: int | None = None++ # `Any` to skip validation, but these are `yarl.URL`. We generate them internally so no validation is needed+ soup_attachments: list[Any] = [] # noqa: RUF012++ def model_post_init(self, *_) -> None:+ if self.published_or_added:+ self.date = to_timestamp(self.published_or_added)++ @property+ def all_files(self) -> Generator[File]:+ if self.file:+ yield self.file+ yield from self.attachments+++class UserPost(Post):+ service: str+ user_id: str = Field(validation_alias="user")+ title: str++ @property+ def user(self) -> User:+ return User(self.service, self.user_id)++ @property+ def web_path_qs(self) -> str:+ return f"{self.service}/user/{self.user_id}/post/{self.id}"+++class DiscordPost(Post):+ server_id: str = Field(validation_alias="server")+ channel_id: str = Field(validation_alias="channel")++ @property+ def web_path_qs(self) -> str:+ return f"discord/server/{self.server_id}/{self.channel_id}#{self.id}"+++class PartialUserPost(NamedTuple):+ """A simplified version of Post that we can built from a soup, for sites that do not have an API.++ Cons: We need to make a request for every individual post"""++ title: str = ""+ content: str = ""+ user_name: str = ""+ date: str | None = None++ @staticmethod+ def from_soup(soup: BeautifulSoup) -> PartialUserPost:+ params = {}+ selectors = (PostSelectors.TITLE, PostSelectors.ALL_CONTENT, PostSelectors.USERNAME, PostSelectors.DATE)+ for name, selector in zip(PartialUserPost._fields, selectors, strict=True):+ if tag := soup.select_one(selector):+ params[name] = tag.get_text().strip()++ return PartialUserPost(**params)+++def fallback_if_no_api(+ func: Callable[Concatenate[KemonoBaseCrawler, _P], Coroutine[None, None, None]],+) -> Callable[Concatenate[KemonoBaseCrawler, _P], Coroutine[None, None, None]]:+ """Calls a fallback method is the current instance does not define an API"""++ @functools.wraps(func)+ async def wrapper(self: KemonoBaseCrawler, *args: _P.args, **kwargs: _P.kwargs) -> None:+ if getattr(self, "API_ENTRYPOINT", None):+ return await func(self, *args, **kwargs)++ if fallback_func := getattr(self, f"{func.__name__}_w_no_api", None):+ return await fallback_func(*args, **kwargs)++ raise ValueError++ return wrapper+++class KemonoBaseCrawler(Crawler, is_abc=True):+ SUPPORTED_PATHS: ClassVar[dict[str, str]] = {+ "Model": "/<service>/user/<user_id>",+ "Favorites": "/favorites/<user_id>",+ "Search": "/search?q=...",+ "Individual Post": "/<service>/user/<user_id>/post/<post_id>",+ "Direct links": "/(data|thumbnail)/...",+ }+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {title}"+ API_ENTRYPOINT: ClassVar[AbsoluteHttpURL]+ SERVICES: ClassVar[tuple[str, ...]] = ()++ def __init_subclass__(cls, **kwargs) -> None:+ super().__init_subclass__(**kwargs)+ Crawler._assert_fields_overrides(cls, "SERVICES")++ def __post_init__(self) -> None:+ self._user_names: dict[User, str] = {}+ self.__known_discord_servers: dict[str, DiscordServer] = {}+ self.__known_attachment_servers: dict[str, str] = {}+ self.__discord_servers_locks: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)++ @property+ def session_cookie(self) -> str:+ return ""++ async def async_startup(self) -> None:+ def check_kemono_page(response: AnyResponse) -> bool:+ if any(x in response.url.parts for x in self.SERVICES):+ return False+ if "discord/channel" in response.url.path:+ return False+ return True++ self.register_cache_filter(self.PRIMARY_URL, check_kemono_page)+ if getattr(self, "API_ENTRYPOINT", None):+ await self._get_usernames(self.API_ENTRYPOINT / "creators")++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case [service, "user", _, "post", _] if service in self.SERVICES:+ return await self.post(scrape_item)+ case [service, "user", _] if service in self.SERVICES:+ return await self.profile(scrape_item)+ case ["favorites", _] if (type_ := scrape_item.url.query.get("type")) in ("post", "artist"):+ return await self.favorites(scrape_item, type_)+ case ["posts"] if search_query := scrape_item.url.query.get("q"):+ return await self.search(scrape_item, search_query)+ case ["discord", "server", server_id]:+ return await self.discord_server(scrape_item, server_id)+ case ["discord", "server", _, channel_id]:+ return await self.discord_channel(scrape_item, channel_id)+ case ["thumbnail" | "thumbnails" | "data", _, *_]:+ return await self.handle_direct_link(scrape_item)+ case _:+ raise ValueError++ @fallback_if_no_api+ @error_handling_wrapper+ async def search(self, scrape_item: ScrapeItem, query: str) -> None:+ api_url = self.__make_api_url_w_offset(scrape_item.url, "posts")+ title = self.create_title(f"{query} [search]")+ scrape_item.setup_as_profile(title)+ await self.__iter_user_posts(scrape_item, api_url)++ @fallback_if_no_api+ @error_handling_wrapper+ async def profile(self, scrape_item: ScrapeItem) -> None:+ path = (scrape_item.url / "posts").path+ api_url = self.__make_api_url_w_offset(scrape_item.url, path)+ scrape_item.setup_as_profile("")+ await self.__iter_user_posts(scrape_item, api_url)++ @fallback_if_no_api+ @error_handling_wrapper+ async def discord_server(self, scrape_item: ScrapeItem, server_id: str) -> None:+ scrape_item.setup_as_forum("")+ server = await self.__get_discord_server(server_id)+ for channel in server.channels:+ url = self.PRIMARY_URL / "discord/server" / server_id / channel.id+ new_scrape_item = scrape_item.create_child(url)+ self.create_task(self.run(new_scrape_item))+ scrape_item.add_children()++ @fallback_if_no_api+ @error_handling_wrapper+ async def discord_channel(self, scrape_item: ScrapeItem, channel_id: str) -> None:+ scrape_item.setup_as_profile("")+ api_url = self.__make_api_url_w_offset(scrape_item.url, f"discord/channel/{channel_id}")+ async for posts in self._pager(api_url, step_size=_DISCORD_CHANNEL_PAGE_SIZE):+ if not isinstance(posts, list):+ error_msg = (+ f"[{self.NAME}] Invalid API response for Discord channel '{channel_id}' posts (URL: {api_url}). "+ f"Expected a list, but got type {type(posts).__name__}. "+ f"Response data (truncated): {str(posts)[:200]}"+ )+ raise ScrapeError(422, error_msg)++ for post_data in posts:+ if not isinstance(post_data, dict):+ error_msg = (+ f"[{self.NAME}] Invalid post data type in list for "+ f"Discord channel '{channel_id}' (URL template: {api_url}). "+ f"Expected a dict for post data, but got type {type(post_data).__name__}. "+ f"Post data (truncated): {str(post_data)[:200]}"+ )+ raise ScrapeError(422, error_msg)++ post = DiscordPost.model_validate(post_data)+ post_web_url = self.parse_url(post.web_path_qs)+ new_scrape_item = scrape_item.create_child(post_web_url)+ self.create_task(self._handle_discord_post_task(new_scrape_item, post))+ scrape_item.add_children()++ if len(posts) < _DISCORD_CHANNEL_PAGE_SIZE:+ break++ @fallback_if_no_api+ @error_handling_wrapper+ async def post(self, scrape_item: ScrapeItem) -> None:+ api_url = self.__make_api_url_w_offset(scrape_item.url)+ json_resp: dict[str, Any] = await self.__api_request(api_url)+ post = UserPost.model_validate(json_resp["post"])+ self._register_attachments_servers(json_resp["attachments"])+ self._handle_user_post(scrape_item, post)++ @fallback_if_no_api+ @error_handling_wrapper+ async def favorites(self, scrape_item: ScrapeItem, type_: Literal["post", "artist"]) -> None:+ if not self.session_cookie:+ msg = "No session cookie found in the config file, cannot scrape favorites"+ raise ScrapeError(401, msg)++ title = f"My favorite {type_}s"+ scrape_item.setup_as_profile(self.create_title(title))+ self.update_cookies({"session": self.session_cookie})+ query_url = (self.API_ENTRYPOINT / "account/favorites").with_query(type=type_)+ resp: list[dict[str, Any]] = await self.__api_request(query_url)+ self.update_cookies({"session": ""})++ for item in resp:+ url = self.PRIMARY_URL / item["service"] / "user" / item["user"]+ if type_ == "post":+ url = url / "post" / item["id"]++ new_scrape_item = scrape_item.create_child(url)+ self.create_task(self.run(new_scrape_item))++ @error_handling_wrapper+ async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:+ scrape_item.url = _thumbnail_to_src(scrape_item.url)+ link = _thumbnail_to_src(url or scrape_item.url)+ try:+ filename, ext = self.get_filename_and_ext(link.query.get("f") or link.name)+ except NoExtensionError:+ # Some patreon URLs have another URL as the filename:+ # ex: https://kemono.su/data/7a...27ad7e40bd.jpg?f=https://www.patreon.com/media-u/Z0F..00672794_+ filename, ext = self.get_filename_and_ext(link.name)+ await self.handle_file(link, scrape_item, link.name, ext, custom_filename=filename)++ # ~~~~~~~~ INTERNAL METHODS, not expected to be overriden, but could be ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++ def _register_attachments_servers(self, attachments: list[File]) -> None:+ for attach in attachments:+ server = attach.get("server")+ if not server:+ continue++ path = attach["path"]+ if previous_server := self.__known_attachment_servers.get(path):+ if previous_server != server:+ msg = (+ f"[{self.NAME}] {path} found with multiple " #+ f"different servers: {server = } {previous_server = } "+ )+ self.log(msg, 30)+ continue+ self.__known_attachment_servers[path] = server++ def _handle_user_post(self, scrape_item: ScrapeItem, post: UserPost) -> None:+ user_name = self._user_names[post.user]+ title = self.create_title(user_name, post.user_id)+ scrape_item.setup_as_album(title, album_id=post.user_id)+ scrape_item.possible_datetime = post.date+ post_title = self.create_separate_post_title(post.title, post.id, post.date)+ scrape_item.add_to_parent_title(post_title)+ self.__handle_post(scrape_item, post)++ async def _handle_discord_post(self, scrape_item: ScrapeItem, post: DiscordPost) -> None:+ server = await self.__get_discord_server(post.server_id)+ title = self.create_title(f"{server.name} [discord]", server.id)+ channel_name = next(c.name for c in server.channels if c.id == post.channel_id)+ scrape_item.setup_as_album(title, album_id=server.id)+ scrape_item.possible_datetime = post.date+ scrape_item.add_to_parent_title(f"#{channel_name}")+ post_title = self.create_separate_post_title(None, post.id, post.date)+ scrape_item.add_to_parent_title(post_title)+ self.__handle_post(scrape_item, post)++ _handle_discord_post_task = auto_task_id(_handle_discord_post)++ def _handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:+ """Gets links out of content in post and sends them to a new crawler."""+ if not post.content:+ return++ for link in self.__parse_content_urls(post):+ new_scrape_item = scrape_item.create_child(link)+ self.handle_external_links(new_scrape_item)+ scrape_item.add_children()++ @error_handling_wrapper+ async def _get_usernames(self, api_url: AbsoluteHttpURL) -> None:+ try:+ json_resp: list[dict[str, Any]] = await self.__api_request(api_url)+ self._user_names = {User(u["service"], u.get("user_id") or u["id"]): u["name"] for u in json_resp}+ except Exception as e:+ msg = f"Unable to get list of creators from {self.NAME}. Crawler has been disabled"+ self.disabled = True+ raise ScrapeError(503, msg) from e++ """~~~~~~~~ PRIVATE METHODS, should never be overriden ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""++ def __parse_content_urls(self, post: Post) -> Generator[AbsoluteHttpURL]:+ seen: set[str] = set()+ for match in _find_http_urls(post.content):+ if (link := match.group().replace(".md.", ".")) not in seen:+ seen.add(link)+ try:+ url = self.parse_url(link)+ except Exception:+ pass+ else:+ if self.DOMAIN not in url.host:+ yield url++ def __handle_post(self, scrape_item: ScrapeItem, post: Post) -> None:+ for file in post.all_files:+ file_url = self.__make_file_url(file)+ self.create_task(self.handle_direct_link(scrape_item, file_url))+ scrape_item.add_children()++ for file_url in post.soup_attachments:+ self.create_task(self.handle_direct_link(scrape_item, file_url))+ scrape_item.add_children()++ self._handle_post_content(scrape_item, post)++ def __make_file_url(self, file: File) -> AbsoluteHttpURL:+ path = file["path"]+ server = self.__known_attachment_servers.get(path) or ""+ url = self.parse_url(server + f"/data{path}")+ return url.with_query(f=file.get("name") or url.name)++ def __make_api_url_w_offset(self, web_url: AbsoluteHttpURL, path: str | None = None) -> AbsoluteHttpURL:+ api_url = self.API_ENTRYPOINT / (path or web_url.path).removeprefix("/")+ offset = int(web_url.query.get("o", 0))+ if query := web_url.query.get("q"):+ return api_url.update_query(o=offset, q=query)+ return api_url.update_query(o=offset)++ async def __get_discord_server(self, server_id: str) -> DiscordServer:+ """Get discord server information, making new API calls if needed."""+ async with self.__discord_servers_locks[server_id]:+ if server := self.__known_discord_servers.get(server_id):+ return server++ server_api_url = self.API_ENTRYPOINT / "discord/user" / server_id / "profile"+ server_profile: dict[str, Any] = await self.__api_request(server_api_url)+ name = server_profile.get("name") or f"Discord Server {server_id}"+ channels_api_url = self.API_ENTRYPOINT / "discord/channel/lookup" / server_id+ channels_resp: list[dict] = await self.__api_request(channels_api_url)+ channels = tuple(DiscordChannel(channel["name"], channel["id"]) for channel in channels_resp)+ self.__known_discord_servers[server_id] = server = DiscordServer(name, server_id, channels)+ return server++ async def __iter_user_posts(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL) -> None:+ async for json_resp in self._pager(url):+ # From search results+ if isinstance(json_resp, dict):+ posts = json_resp.get("posts", [])++ # From profile+ elif isinstance(json_resp, list):+ posts: list[dict[str, Any]] = json_resp++ else:+ raise ScrapeError(422)++ for post in (UserPost.model_validate(entry) for entry in posts):+ post_web_url = self.parse_url(post.web_path_qs)+ new_scrape_item = scrape_item.create_child(post_web_url)+ self._handle_user_post(new_scrape_item, post)+ scrape_item.add_children()++ if len(posts) < _DEFAULT_PAGE_SIZE:+ break++ async def _pager(self, url: AbsoluteHttpURL, step_size: int | None = None) -> AsyncGenerator[Any]:+ """Yields JSON responses from API calls, or soup for web page calls, with configurable increments."""+ current_step_size = step_size or _DEFAULT_PAGE_SIZE+ init_offset = int(url.query.get("o") or 0)++ request = self.__api_request if "api" in url.parts else self.request_soup+ for current_offset in itertools.count(init_offset, current_step_size):+ yield await request(url.update_query(o=current_offset))++ async def __api_request(self, url: AbsoluteHttpURL) -> Any:+ """Get JSON from the API, with a custom Accept header."""++ # When using the 'text/css' header, the response is missing the charset header+ # and charset detection may return a random codec if the body has non english chars, so we force utf-8+ async with self.request(url, headers={"Accept": "text/css"}) as resp:+ return await resp.json(encoding="utf-8", content_type=False)++ # ~~~~~~~~~~ NO API METHODS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++ @error_handling_wrapper+ async def profile_w_no_api(self, scrape_item: ScrapeItem) -> None:+ scrape_item.setup_as_profile("")+ soup: BeautifulSoup+ async for soup in self._pager(scrape_item.url):+ n_posts = 0++ for _, new_scrape_item in self.iter_children(scrape_item, soup, _POST_SELECTOR):+ n_posts += 1+ self.create_task(self.post_w_no_api_task(new_scrape_item))++ if n_posts < _DEFAULT_PAGE_SIZE:+ break++ @error_handling_wrapper+ async def post_w_no_api(self, scrape_item: ScrapeItem) -> None:+ soup = await self.request_soup(scrape_item.url)+ service, _, user_id, _, post_id = scrape_item.url.parts[1:6]+ partial_post = PartialUserPost.from_soup(soup)+ if not partial_post.title or not partial_post.user_name:+ raise ScrapeError(422)++ def files():+ for selector in (+ PostSelectors.VIDEOS,+ PostSelectors.IMAGES,+ PostSelectors.ATTACHMENTS,+ ):+ for file in soup.select(selector):+ yield self.parse_url(css.get_attr(file, "href"))++ post = UserPost(+ user_id=user_id,+ service=service,+ id=post_id,+ title=partial_post.title,+ content=partial_post.content,+ published_or_added=partial_post.date, # type: ignore[reportArgumentType]+ soup_attachments=list(files()),+ )++ self._handle_user_post(scrape_item, post)++ post_w_no_api_task = auto_task_id(post_w_no_api)class KemonoCrawler(KemonoBaseCrawler):SUPPORTED_PATHS: ClassVar = KemonoBaseCrawler.SUPPORTED_PATHS | {"Discord Server": "/discord/<server_id>",- "Discord Server Channel": "/discord/server/...#...",+ "Discord Server Channel": "/discord/server/<server_id>/<channel_id>#...",}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://kemono.cr")API_ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://kemono.cr/api/v1")@@ -27,7 +545,13 @@ class KemonoCrawler(KemonoBaseCrawler):)OLD_DOMAINS: ClassVar[tuple[str, ...]] = "kemono.party", "kemono.su"- async def fetch(self, scrape_item: ScrapeItem) -> None:- if "discord" in scrape_item.url.parts:- return await self.discord(scrape_item)- return await super().fetch(scrape_item)+ @property+ def session_cookie(self):+ return self.manager.config_manager.authentication_data.kemono.session+++def _thumbnail_to_src(og_url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ url = remove_parts(og_url, "thumbnails", "thumbnail").with_query(None)+ if name := og_url.query.get("f"):+ return url.with_query(f=name)+ return urldiff --git a/cyberdrop_dl/crawlers/nekohouse.py b/cyberdrop_dl/crawlers/nekohouse.pyindex 758300a09..728fca195 100644--- a/cyberdrop_dl/crawlers/nekohouse.py+++ b/cyberdrop_dl/crawlers/nekohouse.py@@ -1,33 +1,24 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import ClassVarfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from ._kemono_base import KemonoBaseCrawler--if TYPE_CHECKING:- from cyberdrop_dl.data_structures.url_objects import ScrapeItem+from .kemono import KemonoBaseCrawlerclass NekohouseCrawler(KemonoBaseCrawler):SUPPORTED_PATHS: ClassVar[dict[str, str]] = {- "Model": "/<service>/user/<user>",- "Individual Post": "/<service>/<user>/post/...",- "Direct links": "",+ "Model": "/<service>/user/<user_id>",+ "Individual Post": "/<service>/user/<user_id>/post/<post_id>",+ "Direct links": "/(data|thumbnails)/...",}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://nekohouse.su")DOMAIN: ClassVar[str] = "nekohouse"SERVICES = "fanbox", "fantia", "fantia_products", "subscribestar", "twitter"- async def fetch(self, scrape_item: ScrapeItem) -> None:- if "thumbnails" in scrape_item.url.parts:- return await self.handle_direct_link(scrape_item)- if "post" in scrape_item.url.parts:- return await self.post_w_no_api(scrape_item)- if any(x in scrape_item.url.parts for x in self.SERVICES):- return await self.profile_w_no_api(scrape_item)- if any(x in scrape_item.url.parts for x in ("posts", "discord")):- raise ValueError+ async def async_startup(self) -> None:+ await super().async_startup()- await self.handle_direct_link(scrape_item)+ # Only this API endpoint is available+ await self._get_usernames(self.PRIMARY_URL / "api/creators")diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 3b9927264..b7d16d146 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -181,7 +181,7 @@ class Downloader:self.update_queued_files()task_id = self.manager.progress_manager.file_progress.add_task(domain=self.domain, filename=media_item.filename)media_item.set_task_id(task_id)- video, audio, subtitles = await self._download_rendition_group(media_item, m3u8_group)+ video, audio, _ = await self._download_rendition_group(media_item, m3u8_group)if not audio:await asyncio.to_thread(video.rename, media_item.complete_file)else:diff --git a/tests/crawlers/test_cases/coomer.py b/tests/crawlers/test_cases/coomer.pynew file mode 100644index 000000000..7ada225ab--- /dev/null+++ b/tests/crawlers/test_cases/coomer.py@@ -0,0 +1,41 @@+DOMAIN = "coomer"+TEST_CASES = [+ (+ "https://coomer.st/fansly/user/445978322480930816/post/644032890815913985",+ [+ {+ "url": "https://coomer.st/data/03/ef/03ef02264303b2693c2acb714cf0321d3bfca04b85511dc4f66887ad134cb69e.jpg?f=644032704089694208.jpeg",+ "filename": "644032704089694208.jpeg",+ "referer": "https://coomer.st/fansly/user/445978322480930816/post/644032890815913985",+ "download_folder": r"re:zombonn \(Coomer\)",+ "album_id": "445978322480930816",+ "datetime": 1715061766,+ }+ ],+ ),+ (+ "https://coomer.st/onlyfans/user/lostfaefoundskates/post/729202326",+ [+ {+ "url": "https://coomer.st/data/fe/af/feafbd06f78da02309df109c6ed4b83dd877cc256ecfc41212d5adf8673166a6.jpg?f=3840x2880_ccd7f56947c484db31c59421343c71cd.jpg",+ "filename": "3840x2880_ccd7f56947c484db31c59421343c71cd.jpg",+ "referer": "https://coomer.st/onlyfans/user/lostfaefoundskates/post/729202326",+ "download_folder": r"re:lostfaefoundskates \(Coomer\)",+ "album_id": "lostfaefoundskates",+ "datetime": 1694414991,+ }+ ],+ ),+ (+ "https://coomer.st/data/4b/de/4bde70ea2dd6284938aa90af959f78c4f3e24fbf6b06ed842f0356824fe9bef0.jpg?f=3024x4032_c0ef341d7e60ceed82a906b8062806cb.jpg",+ [+ {+ "url": "https://coomer.st/data/4b/de/4bde70ea2dd6284938aa90af959f78c4f3e24fbf6b06ed842f0356824fe9bef0.jpg?f=3024x4032_c0ef341d7e60ceed82a906b8062806cb.jpg",+ "filename": "3024x4032_c0ef341d7e60ceed82a906b8062806cb.jpg",+ "referer": "https://coomer.st/data/4b/de/4bde70ea2dd6284938aa90af959f78c4f3e24fbf6b06ed842f0356824fe9bef0.jpg?f=3024x4032_c0ef341d7e60ceed82a906b8062806cb.jpg",+ "album_id": None,+ "datetime": None,+ },+ ],+ ),+]diff --git a/tests/crawlers/test_cases/kemono.py b/tests/crawlers/test_cases/kemono.pyold mode 100644new mode 100755index 673d93993..7f0d61b39--- a/tests/crawlers/test_cases/kemono.py+++ b/tests/crawlers/test_cases/kemono.py@@ -1,15 +1,65 @@DOMAIN = "kemono"TEST_CASES = [(- "https://kemono.su/patreon/user/27174868/post/133667042",+ "https://kemono.cr/patreon/user/27174868/post/133667042",[{- "url": "https://kemono.su/data/0d/c3/0dc3ecec556918b882674c5dd63f8fa1867867157ce1ca675b0bf79e6b73fb04.jpg?f=Birthday_Kim.jpg",+ "url": "https://kemono.cr/data/0d/c3/0dc3ecec556918b882674c5dd63f8fa1867867157ce1ca675b0bf79e6b73fb04.jpg?f=Birthday_Kim.jpg","filename": "Birthday_Kim.jpg",- "referer": "https://kemono.su/patreon/user/27174868/post/133667042",+ "referer": "https://kemono.cr/patreon/user/27174868/post/133667042","album_id": "27174868","datetime": 1752004344,}],- )+ ),+ (+ "https://kemono.cr/data/49/83/49830d8d281ad9a508f88b081b3a112aa4011a1af3bb2539f88f4e635edf98a1.png?f=june_Schedule.png",+ [+ {+ "url": "https://kemono.cr/data/49/83/49830d8d281ad9a508f88b081b3a112aa4011a1af3bb2539f88f4e635edf98a1.png?f=june_Schedule.png",+ "filename": "june_Schedule.png",+ "referer": "https://kemono.cr/data/49/83/49830d8d281ad9a508f88b081b3a112aa4011a1af3bb2539f88f4e635edf98a1.png?f=june_Schedule.png",+ "album_id": None,+ "datetime": None,+ },+ ],+ ),+ (+ "https://kemono.cr/discord/server/794379082535927828/921981855706796032",+ [+ {+ "url": "https://kemono.cr/data/49/83/49830d8d281ad9a508f88b081b3a112aa4011a1af3bb2539f88f4e635edf98a1.png?f=june_Schedule.png",+ "filename": "june_Schedule.png",+ "referer": "https://kemono.cr/discord/server/794379082535927828/921981855706796032#1257906262025175090",+ "download_folder": r"re:Schpicy-s server \[discord\] \(Kemono\)\/\#schedule",+ "album_id": "794379082535927828",+ "datetime": 1719996623,+ },+ ],+ 3,+ ),+ (+ "https://kemono.cr/patreon/user/16573132",+ [+ {+ "url": "https://kemono.cr/data/01/1e/011e7a27eca93eba99ab223553d6079495a55c87106fa0e36efb8810048da92f.jpg?f=httpswww.patreon.commedia-uZ0FBQUFBQmdWWnhiV0xoYnpvV0Nnd0ZvMnM5UnkyTVF1LXd4MEpNYmVRWGNMNEZhUE1wWWJreVBqS1F2UzB3R2lYaXpDaXBWUy14M2l1WGtITWJBX0Q4ZDRoQzBtOG5nd3JIYThERTRWNDIxeUY4RHBXaERxWkRJNS1xRXRPS2lFM3pLSEVTSW5oNVRzeU9yc21SMmo5MkJrYXVXVzFCVUtRPT0",+ "filename": "011e7a27eca93eba99ab223553d6079495a55c87106fa0e36efb8810048da92f.jpg",+ "referer": "https://kemono.cr/patreon/user/16573132/post/42836761",+ "download_folder": r"re:EUNSONGS \(Kemono\)",+ "album_id": "16573132",+ "datetime": 1602954002,+ }+ ],+ 212,+ ),+ (+ "https://kemono.cr/posts?q=dandadan%20s2%20episode%208%20reaction",+ [+ {+ "url": "ANY",+ "download_folder": r"re:dandadan s2 episode 8 reaction \[search\] \(Kemono\)",+ }+ ],+ 43,+ ),]diff --git a/tests/crawlers/test_cases/nekohouse.py b/tests/crawlers/test_cases/nekohouse.pynew file mode 100644index 000000000..4f32a8bbd--- /dev/null+++ b/tests/crawlers/test_cases/nekohouse.py@@ -0,0 +1,56 @@+DOMAIN = "nekohouse"+TEST_CASES = [+ (+ "https://nekohouse.su/fanbox/user/705370/post/9605906",+ [+ {+ "url": "https://nekohouse.su/data/33/7f/337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg",+ "filename": "337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg",+ "referer": "https://nekohouse.su/fanbox/user/705370/post/9605906",+ "download_folder": r"re:shiratamaco \(Nekohouse\)",+ "album_id": "705370",+ "datetime": 1743063817,+ },+ {+ "url": "https://nekohouse.su/data/ec/7b/ec7bf13520488ea9ed19e766eeac6033547e69eca88e395f12c9e503feaa71a5.jpg",+ "filename": "ec7bf13520488ea9ed19e766eeac6033547e69eca88e395f12c9e503feaa71a5.jpg",+ "referer": "https://nekohouse.su/fanbox/user/705370/post/9605906",+ "album_id": "705370",+ "datetime": 1743063817,+ },+ {+ "url": "https://nekohouse.su/data/fa/66/fa66c78e0fefaa748baeee23dafc5392236b3a194aa0f2b3040dced30a617b7f.jpg",+ "filename": "fa66c78e0fefaa748baeee23dafc5392236b3a194aa0f2b3040dced30a617b7f.jpg",+ "referer": "https://nekohouse.su/fanbox/user/705370/post/9605906",+ "album_id": "705370",+ "datetime": 1743063817,+ },+ ],+ ),+ (+ "https://nekohouse.su/thumbnail/data/33/7f/337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg?type=preview",+ [+ {+ "url": "https://nekohouse.su/data/33/7f/337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg",+ "filename": "337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg",+ "referer": "https://nekohouse.su/data/33/7f/337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg",+ "album_id": None,+ "datetime": None,+ },+ ],+ ),+ (+ "https://nekohouse.su/fanbox/user/92778036",+ [+ {+ "url": "https://nekohouse.su/data/01/44/014495e8f29a8a31ac008ff0038392f738a5237eff36c06a29b35f60d9ef470e.png",+ "filename": "014495e8f29a8a31ac008ff0038392f738a5237eff36c06a29b35f60d9ef470e.png",+ "referer": "https://nekohouse.su/fanbox/user/92778036/post/9570757",+ "download_folder": r"re:redkingtaraba \(Nekohouse\)",+ "album_id": "92778036",+ "datetime": 1742473273,+ },+ ],+ 549,+ ),+]