Skip to content

Bulk asynchronous downloader for multiple file hosts

perf: skip duplicate URLs (Kemono) (#1220)

NTFSvolume61515883213d
diff --git a/cyberdrop_dl/crawlers/_kemono_base.py b/cyberdrop_dl/crawlers/_kemono_base.py
deleted file mode 100644
index 572c716fa..000000000
--- a/cyberdrop_dl/crawlers/_kemono_base.py
+++ /dev/null
@@ -1,596 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import functools
-import itertools
-import re
-from collections import defaultdict
-from datetime import datetime # noqa: TC003
-from json import loads as json_loads
-from typing import TYPE_CHECKING, Annotated, Any, ClassVar, NamedTuple, NotRequired, cast
-
-from pydantic import AliasChoices, BeforeValidator, Field
-from typing_extensions import TypedDict # Import from typing is not compatible with pydantic
-
-from cyberdrop_dl.crawlers.crawler import Crawler
-from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError
-from cyberdrop_dl.models import AliasModel
-from cyberdrop_dl.models.validators import falsy_as_none
-from cyberdrop_dl.utils import css
-from cyberdrop_dl.utils.dates import to_timestamp
-from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts
-
-if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
-
- from aiohttp_client_cache.response import AnyResponse
- from bs4 import BeautifulSoup
-
- from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
-
-
-MAX_OFFSET_PER_CALL = 50
-DISCORD_CHANNEL_PAGE_SIZE = 150
-LINK_REGEX = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))")
-POST_SELECTOR = "article.post-card a"
-
-
-class PostSelectors:
- FILES = "div[class*=__files]"
- CONTENT = "div[class*=__content]"
- ATTACHMENTS = "a[class*=__attachment-link]"
- IMAGES = "div[class=fileThumb]"
- VIDEOS = "video[class*=__video] source"
- ALL_CONTENT = f"{FILES}, {CONTENT}"
-
- DATE_PUBLISHED = "div[class*=__published] time[class=timestamp]"
- DATE_ADDED = "div[class*=__added]"
- DATE = f"{DATE_PUBLISHED}, {DATE_ADDED}"
-
- TITLE = "h1[class*=__title]"
- USERNAME = "a[class*=__user-name]"
-
-
-_POST = PostSelectors()
-
-
-class UserURL(NamedTuple):
- # format: https://kemono.su/<service>/user/<user_id>/post/<post_id>
- # ex: https://kemono.su/patreon/user/92916478/post/87864845
- service: str
- user_id: str
- post_id: str | None = None
-
- @staticmethod
- def parse(url: AbsoluteHttpURL) -> UserURL:
- if (n_parts := len(url.parts)) > 3:
- post_id = url.parts[5] if n_parts > 5 else None
- return UserURL(url.parts[1], url.parts[3], post_id)
- msg = "Invalid user URL"
- raise ValueError(msg)
-
-
-class UserPostURL(UserURL):
- post_id: str
-
- @staticmethod
- def parse(url: AbsoluteHttpURL) -> UserPostURL:
- result = UserURL.parse(url)
- assert result.post_id, "Individual posts must have a post_id"
- return cast("UserPostURL", result)
-
-
-class DiscordURL(NamedTuple):
- # format: https://kemono.su/discord/server/<server_id>/<channel_id>
- # ex: https://kemono.su/discord/server/891670433978531850/892624523034255371
- server_id: str
- channel_id: str | None = None # Only present for individual channels URLs
-
- @staticmethod
- def parse(url: AbsoluteHttpURL) -> DiscordURL:
- return DiscordURL(*url.parts[3:5])
-
-
-class DiscordChannel(NamedTuple):
- name: str
- id: str
-
-
-class DiscordServer(NamedTuple):
- name: str
- id: str
- channels: tuple[DiscordChannel, ...]
-
-
-class User(NamedTuple):
- service: str
- id: str
-
-
-class File(TypedDict):
- name: NotRequired[str] # Sometimes present
- path: str
- server: NotRequired[str] # Sometimes present in attachments
-
-
-FileOrNone = Annotated[File | None, BeforeValidator(falsy_as_none)]
-
-
-class Post(AliasModel):
- id: str
- content: str = ""
- file: FileOrNone = None
- attachments: list[File] = [] # noqa: RUF012
- published_or_added: datetime | None = Field(None, validation_alias=AliasChoices("published", "added"))
- soup_attachments: list[Any] = [] # noqa: RUF012, `Any` to skip validation, but these are `yarl.URL`. We generate them internally so no validation is needed
- date: int | None = None
-
- def model_post_init(self, *_) -> None:
- if self.published_or_added:
- self.date = to_timestamp(self.published_or_added)
-
- @property
- def all_files(self) -> Generator[File]:
- if self.file:
- yield self.file
- yield from self.attachments
-
-
-class UserPost(Post):
- service: str
- user_id: str = Field(validation_alias="user")
- title: str
-
- @property
- def user(self) -> User:
- return User(self.service, self.user_id)
-
- @property
- def web_path_qs(self) -> str:
- return f"{self.service}/user/{self.user_id}/post/{self.id}"
-
-
-class DiscordPost(Post):
- server_id: str = Field(validation_alias="server")
- channel_id: str = Field(validation_alias="channel")
-
- @property
- def web_path_qs(self) -> str:
- return f"discord/server/{self.server_id}/{self.channel_id}#{self.id}"
-
-
-class PartialUserPost(NamedTuple):
- """A simplified version of Post that we can built from a soup, for sites that do not have an API
-
- Pros: We can get the post data + user_name in a single request
-
- Cons: We need to make a request for every individual post"""
-
- title: str = ""
- content: str = ""
- user_name: str = ""
- date: str | None = None
-
- @staticmethod
- def from_soup(soup: BeautifulSoup) -> PartialUserPost:
- info = {}
- names = "title", "content", "user_name", "date"
- selectors = (_POST.TITLE, _POST.ALL_CONTENT, _POST.USERNAME, _POST.DATE)
- for name, selector in zip(names, selectors, strict=True):
- if tag := soup.select_one(selector):
- info[name] = tag.text.strip()
-
- return PartialUserPost(**info)
-
-
-def fallback_if_no_api(func: Callable[..., Coroutine[None, None, Any]]) -> Callable[..., Coroutine[None, None, Any]]:
- """Calls a fallback method is the current instance does not define an API"""
-
- @functools.wraps(func)
- async def wrapper(self: KemonoBaseCrawler, *args, **kwargs) -> Any:
- if hasattr(self, "API_ENTRYPOINT"):
- return await func(self, *args, **kwargs)
- fallback_func = getattr(self, f"{func.__name__}_w_no_api", None)
- if not fallback_func:
- raise ValueError
- return await fallback_func(self, *args, **kwargs)
-
- return wrapper
-
-
-class KemonoBaseCrawler(Crawler, is_abc=True):
- SUPPORTED_PATHS: ClassVar[dict[str, str]] = {
- "Model": "/<service>/user/<user>",
- "Favorites": "/favorites/<user>",
- "Search": "/search?...",
- "Individual Post": "<service>/<user>/post/...",
- "Direct links": "",
- }
- DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {title}"
- API_ENTRYPOINT: ClassVar[AbsoluteHttpURL]
- SERVICES: ClassVar[tuple[str, ...]] = ()
-
- def __init_subclass__(cls, **kwargs) -> None:
- super().__init_subclass__(**kwargs)
- REQUIRED_FIELDS = ("SERVICES",)
- for field_name in REQUIRED_FIELDS:
- assert getattr(cls, field_name, None), f"Subclass {cls.__name__} must override: {field_name}"
-
- def __post_init__(self) -> None:
- self._user_names: dict[User, str] = {}
- self.__known_discord_servers: dict[str, DiscordServer] = {}
- self.__known_attachment_servers: dict[str, str] = {}
- self.__discord_servers_locks: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
- self.session_cookie = self.manager.config_manager.authentication_data.kemono.session
-
- async def async_startup(self) -> None:
- def check_kemono_page(response: AnyResponse) -> bool:
- if any(x in response.url.parts for x in self.SERVICES):
- return False
- if "discord/channel" in response.url.path:
- return False
- return True
-
- self.register_cache_filter(self.PRIMARY_URL, check_kemono_page)
- if getattr(self, "API_ENTRYPOINT", None):
- await self.__get_usernames(self.API_ENTRYPOINT / "creators")
-
- async def fetch(self, scrape_item: ScrapeItem) -> None:
- if "thumbnails" in scrape_item.url.parts:
- return await self.handle_direct_link(scrape_item)
- if "post" in scrape_item.url.parts:
- return await self.post(scrape_item)
- if scrape_item.url.name == "posts":
- if not scrape_item.url.query.get("q"):
- raise ValueError
- return await self.search(scrape_item)
- if any(x in scrape_item.url.parts for x in self.SERVICES):
- return await self.profile(scrape_item)
- if "favorites" in scrape_item.url.parts:
- return await self.favorites(scrape_item)
- await self.handle_direct_link(scrape_item)
-
- @fallback_if_no_api
- @error_handling_wrapper
- async def search(self, scrape_item: ScrapeItem) -> None:
- query = scrape_item.url.query["q"]
- api_url = self.__make_api_url_w_offset("posts", scrape_item.url)
- title = self.create_title(f"{query} [search]")
- scrape_item.setup_as_profile(title)
- await self.__iter_user_posts_from_url(scrape_item, api_url)
-
- @fallback_if_no_api
- @error_handling_wrapper
- async def profile(self, scrape_item: ScrapeItem) -> None:
- url_info = UserURL.parse(scrape_item.url)
- path = f"{url_info.service}/user/{url_info.user_id}/posts"
- api_url = self.__make_api_url_w_offset(path, scrape_item.url)
- scrape_item.setup_as_profile("")
- await self.__iter_user_posts_from_url(scrape_item, api_url)
-
- @fallback_if_no_api
- @error_handling_wrapper
- async def discord(self, scrape_item: ScrapeItem) -> None:
- discord = DiscordURL.parse(scrape_item.url)
- if discord.channel_id:
- return await self.discord_channel(scrape_item, discord.channel_id)
-
- await self.discord_server(scrape_item, discord.server_id)
-
- async def discord_server(self, scrape_item: ScrapeItem, server_id: str) -> None:
- scrape_item.setup_as_forum("")
- server = await self.__get_discord_server(server_id)
- for channel in server.channels:
- url = self.PRIMARY_URL / "discord/server" / server_id / channel.id
- new_scrape_item = scrape_item.create_child(url)
- self.create_task(self.run(new_scrape_item))
- scrape_item.add_children()
-
- async def discord_channel(self, scrape_item: ScrapeItem, channel_id: str) -> None:
- scrape_item.setup_as_profile("")
- api_url_template = self.__make_api_url_w_offset(f"discord/channel/{channel_id}", scrape_item.url)
- async for json_response_list in self.__api_pager(api_url_template, step_size=DISCORD_CHANNEL_PAGE_SIZE):
- if not isinstance(json_response_list, list):
- error_msg = (
- f"[{self.NAME}] Invalid API response for Discord channel '{channel_id}' posts (URL template: {api_url_template}). "
- f"Expected a list, but got type {type(json_response_list).__name__}. "
- f"Response data (truncated): {str(json_response_list)[:200]}"
- )
- raise ScrapeError(422, error_msg)
- if not json_response_list:
- break
-
- num_posts_in_page = 0
- for post_data in json_response_list:
- num_posts_in_page += 1
- if not isinstance(post_data, dict):
- error_msg = (
- f"[{self.NAME}] Invalid post data type in list for Discord channel '{channel_id}' (URL template: {api_url_template}). "
- f"Expected a dict for post data, but got type {type(post_data).__name__}. "
- f"Post data (truncated): {str(post_data)[:200]}"
- )
- raise ScrapeError(422, error_msg)
- post = DiscordPost.model_validate(post_data)
- post_web_url = self.parse_url(post.web_path_qs)
- new_scrape_item_for_post = scrape_item.create_child(post_web_url)
- await self._handle_discord_post(new_scrape_item_for_post, post)
- scrape_item.add_children()
-
- if num_posts_in_page < DISCORD_CHANNEL_PAGE_SIZE:
- break
-
- @fallback_if_no_api
- @error_handling_wrapper
- async def post(self, scrape_item: ScrapeItem) -> None:
- url_info = UserPostURL.parse(scrape_item.url)
- path = f"{url_info.service}/user/{url_info.user_id}/post/{url_info.post_id}"
- api_url = self.__make_api_url_w_offset(path, scrape_item.url)
- json_resp: dict[str, Any] = await self.__api_request(api_url)
- post = UserPost.model_validate(json_resp["post"])
- self._register_attachments_servers(json_resp["attachments"])
- await self._handle_user_post(scrape_item, post)
-
- def _register_attachments_servers(self, attachments: list[File]) -> None:
- for attach in attachments:
- if server := attach.get("server"):
- path = attach["path"]
- if previous_server := self.__known_attachment_servers.get(path):
- if previous_server != server:
- msg = f"[{self.NAME}] {path} found with multiple diferent servers: {server = } {previous_server = } "
- self.log_debug(msg)
- continue
- self.__known_attachment_servers[path] = server
-
- @error_handling_wrapper
- async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:
- """Handles a direct link."""
-
- def clean_url(og_url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- if "thumbnails" in og_url.parts:
- return remove_parts(og_url, "thumbnails")
- return og_url
-
- scrape_item.url = clean_url(scrape_item.url)
- link = clean_url(url or scrape_item.url)
- try:
- filename, ext = self.get_filename_and_ext(link.query.get("f") or link.name)
- except NoExtensionError:
- # Some patreon URLs have another URL as the filename: https://kemono.su/data/7a...27ad7e40bd.jpg?f=https://www.patreon.com/media-u/Z0F..00672794_
- filename, ext = self.get_filename_and_ext(link.name)
- await self.handle_file(link, scrape_item, filename, ext)
-
- @error_handling_wrapper
- async def favorites(self, scrape_item: ScrapeItem) -> None:
- if not self.session_cookie:
- raise ScrapeError(401, "No session cookie found in the config file, cannot scrape favorites")
-
- is_post = scrape_item.url.query.get("type") == "post"
- title = "My favorite posts" if is_post else "My favorite artists"
- scrape_item.setup_as_profile(self.create_title(title))
-
- self.update_cookies({"session": self.session_cookie})
- api_url = self.API_ENTRYPOINT / "account/favorites"
- query_url = api_url.with_query(type="post" if is_post else "artist")
- json_resp: list[dict] = await self.__api_request(query_url)
- self.update_cookies({"session": ""})
-
- for item in json_resp:
- if is_post:
- post_id, user_id, service = item["id"], item["user"], item["service"]
- url = self.PRIMARY_URL / service / "user" / user_id / "post" / post_id
- else:
- user_id, service = item["id"], item["service"]
- url = self.PRIMARY_URL / service / "user" / user_id
-
- new_scrape_item = scrape_item.create_child(url)
- self.create_task(self.run(new_scrape_item))
-
- # ~~~~~~~~ INTERNAL METHODS, not expected to be overriden, but could be ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
- async def _handle_user_post(self, scrape_item: ScrapeItem, post: UserPost):
- user_name = self._user_names[post.user]
- title = self.create_title(user_name, post.user_id)
- scrape_item.setup_as_album(title, album_id=post.user_id)
- scrape_item.possible_datetime = post.date
- post_title = self.create_separate_post_title(post.title, post.id, post.date)
- scrape_item.add_to_parent_title(post_title)
-
- await self.__handle_post(scrape_item, post)
-
- async def _handle_discord_post(self, scrape_item: ScrapeItem, post: DiscordPost):
- server = await self.__get_discord_server(post.server_id)
- title = self.create_title(f"{server.name} [discord]", server.id)
- channel_name = next(c.name for c in server.channels if c.id == post.channel_id)
- scrape_item.setup_as_album(title, album_id=server.id)
- scrape_item.possible_datetime = post.date
- scrape_item.add_to_parent_title(f"#{channel_name}")
-
- post_title = self.create_separate_post_title(None, post.id, post.date)
- scrape_item.add_to_parent_title(post_title)
-
- await self.__handle_post(scrape_item, post)
-
- def _handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:
- """Gets links out of content in post ans sends them to a new crawler."""
- if not post.content:
- return
-
- def gen_yarl_urls() -> Generator[AbsoluteHttpURL]:
- for match in re.finditer(LINK_REGEX, post.content):
- link = match.group().replace(".md.", ".")
- try:
- url = self.parse_url(link)
- yield url
- except Exception:
- pass
-
- for link in gen_yarl_urls():
- if not link.host or self.DOMAIN in link.host:
- continue
- new_scrape_item = scrape_item.create_child(link)
- self.handle_external_links(new_scrape_item)
- scrape_item.add_children()
-
- """~~~~~~~~ PRIVATE METHODS, should never be overriden ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
-
- async def __handle_post(self, scrape_item: ScrapeItem, post: Post) -> None:
- # Process files if the post was generated from an API call
- for file in post.all_files:
- file_url = self.__make_file_url(file)
- await self.handle_direct_link(scrape_item, file_url)
- scrape_item.add_children()
-
- # Process files if the post was generated from soup
- for file_url in post.soup_attachments:
- await self.handle_direct_link(scrape_item, file_url)
- scrape_item.add_children()
-
- self._handle_post_content(scrape_item, post)
-
- def __make_file_url(self, file: File) -> AbsoluteHttpURL:
- server = self.__known_attachment_servers.get(file["path"], "")
- url = self.parse_url(server + f"/data{file['path']}")
- return url.with_query(f=file.get("name") or url.name)
-
- def __make_api_url_w_offset(self, path: str, og_url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- api_url = self.API_ENTRYPOINT / path
- offset = int(og_url.query.get("o", 0))
- if query := og_url.query.get("q"):
- return api_url.update_query(o=offset, q=query)
- return api_url.update_query(o=offset)
-
- async def __get_usernames(self, api_url: AbsoluteHttpURL) -> None:
- try:
- json_resp: list[dict[str, Any]] = await self.__api_request(api_url)
- self._user_names = {User(u["service"], u["id"]): u["name"] for u in json_resp}
- except Exception:
- pass
- if not self._user_names:
- self.log(f"Unable to get list of creators from {self.NAME}. Crawler has been disabled")
- self.disabled = True
-
- async def __get_discord_server(self, server_id: str) -> DiscordServer:
- """Get discord server information, making new API calls if needed."""
- async with self.__discord_servers_locks[server_id]:
- if server := self.__known_discord_servers.get(server_id):
- return server
-
- api_url_server_profile = self.API_ENTRYPOINT / "discord" / "user" / server_id / "profile"
- server_profile_json: dict[str, Any] = await self.__api_request(api_url_server_profile)
- name = server_profile_json.get("name")
- if not name:
- name = f"Discord Server {server_id}"
- api_url_channels = self.API_ENTRYPOINT / "discord/channel/lookup" / server_id
- channels_json_resp: list[dict] = await self.__api_request(api_url_channels)
- channels = tuple(DiscordChannel(channel["name"], channel["id"]) for channel in channels_json_resp)
- self.__known_discord_servers[server_id] = server = DiscordServer(name, server_id, channels)
- return server
-
- async def __iter_user_posts_from_url(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL) -> None:
- async for json_resp in self.__api_pager(url):
- n_posts = 0
-
- # From search results
- if isinstance(json_resp, dict):
- posts = json_resp.get("posts", [])
- # From profile
- elif isinstance(json_resp, list):
- posts: list[dict[str, Any]] = json_resp
- else:
- raise ScrapeError(422)
-
- if not posts:
- return
-
- for post in (UserPost.model_validate(entry) for entry in posts):
- n_posts += 1
- link = self.parse_url(post.web_path_qs)
- new_scrape_item = scrape_item.create_child(link)
- await self._handle_user_post(new_scrape_item, post)
- scrape_item.add_children()
-
- if n_posts < MAX_OFFSET_PER_CALL:
- break
-
- async def __api_pager(self, url: AbsoluteHttpURL, step_size: int | None = None) -> AsyncGenerator[Any, None]:
- """Yields JSON responses from API calls, with configurable increments."""
- current_step_size = step_size if step_size is not None else MAX_OFFSET_PER_CALL
- init_offset = int(url.query.get("o") or 0)
- for current_offset in itertools.count(init_offset, current_step_size):
- api_url = url.update_query(o=current_offset)
- yield await self.__api_request(api_url)
-
- # ~~~~~~~~~~ NO API METHODS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
- async def discord_w_no_api(self, scrape_item: ScrapeItem) -> Any:
- raise NotImplementedError
-
- async def search_w_no_api(self, scrape_item: ScrapeItem) -> Any:
- raise NotImplementedError
-
- @error_handling_wrapper
- async def profile_w_no_api(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
-
- url_info = UserURL.parse(scrape_item.url)
- path = f"{url_info.service}/user/{url_info.user_id}"
- api_url = self.PRIMARY_URL / path
- scrape_item.setup_as_profile("")
- init_offset = int(scrape_item.url.query.get("o") or 0)
- for offset in itertools.count(init_offset, MAX_OFFSET_PER_CALL):
- n_posts = 0
- api_url = api_url.with_query(o=offset)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, api_url)
-
- for post in soup.select(POST_SELECTOR):
- n_posts += 1
- link = self.parse_url(css.get_attr(post, "href"))
- new_scrape_item = scrape_item.create_child(link)
- await self.post_w_no_api(new_scrape_item)
- scrape_item.add_children()
-
- if n_posts < MAX_OFFSET_PER_CALL:
- break
-
- @error_handling_wrapper
- async def post_w_no_api(self, scrape_item: ScrapeItem) -> None:
- url_info = UserPostURL.parse(scrape_item.url)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
-
- post = PartialUserPost.from_soup(soup)
- if not post.title or not post.user_name:
- raise ScrapeError(422)
-
- files: list[AbsoluteHttpURL] = []
- for selector in (_POST.VIDEOS, _POST.IMAGES, _POST.ATTACHMENTS):
- for file in soup.select(selector):
- files.append(self.parse_url(css.get_attr(file, "href")))
-
- full_post = UserPost(
- user_id=url_info.user_id,
- service=url_info.service,
- id=url_info.post_id,
- title=post.title,
- content=post.content,
- published_or_added=post.date, # type: ignore[reportArgumentType]
- soup_attachments=files,
- )
-
- self._user_names[full_post.user] = post.user_name
- await self._handle_user_post(scrape_item, full_post)
-
- async def __api_request(self, url: AbsoluteHttpURL) -> Any:
- """Get JSON from the API, with a custom Accept header."""
-
- async with self.request_limiter:
- response, soup_or_none = await self.client._get(self.DOMAIN, url, headers={"Accept": "text/css"})
-
- if soup_or_none:
- return json_loads(soup_or_none.text)
-
- # When using the 'text/css' header, the response is missing the charset header
- # and charset detection may return a random codec if the body has non english chars, so we force utf-8
- return json_loads(await response.text(encoding="utf-8"))
diff --git a/cyberdrop_dl/crawlers/coomer.py b/cyberdrop_dl/crawlers/coomer.py
index 29ce20c62..6594734f6 100644
--- a/cyberdrop_dl/crawlers/coomer.py
+++ b/cyberdrop_dl/crawlers/coomer.py
@@ -2,11 +2,9 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
-from aiolimiter import AsyncLimiter
-
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from ._kemono_base import KemonoBaseCrawler, Post
+from .kemono import KemonoBaseCrawler, Post
if TYPE_CHECKING:
from aiohttp_client_cache.response import AnyResponse
@@ -21,10 +19,9 @@ class CoomerCrawler(KemonoBaseCrawler):
SERVICES = "onlyfans", "fansly"
OLD_DOMAINS: ClassVar[tuple[str, ...]] = "coomer.party", "coomer.su"
- def __post_init__(self) -> None:
- super().__post_init__()
- self.request_limiter = AsyncLimiter(4, 1)
- self.session_cookie = self.manager.config_manager.authentication_data.coomer.session
+ @property
+ def session_cookie(self) -> str:
+ return self.manager.config_manager.authentication_data.coomer.session
async def async_startup(self) -> None:
await super().async_startup()
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 2276568c0..31ee0b25c 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -150,6 +150,12 @@ class Crawler(ABC):
def __post_init__(self) -> None: ... # noqa: B027
+ @final
+ @staticmethod
+ def _assert_fields_overrides(subclass: type[Crawler], *fields: str):
+ for field_name in fields:
+ assert getattr(subclass, field_name, None), f"Subclass {subclass.__name__} must override: {field_name}"
+
def __init_subclass__(
cls, is_abc: bool = False, is_generic: bool = False, generic_name: str = "", **kwargs
) -> None:
@@ -177,9 +183,7 @@ class Crawler(ABC):
return
if not (cls.IS_FALLBACK_GENERIC or cls.IS_REAL_DEBRID):
- REQUIRED_FIELDS = "PRIMARY_URL", "DOMAIN", "SUPPORTED_PATHS"
- for field_name in REQUIRED_FIELDS:
- assert getattr(cls, field_name, None), f"Subclass {cls.__name__} must override: {field_name}"
+ Crawler._assert_fields_overrides(cls, "PRIMARY_URL", "DOMAIN", "SUPPORTED_PATHS")
if cls.OLD_DOMAINS:
cls.REPLACE_OLD_DOMAINS_REGEX = re.compile("|".join(cls.OLD_DOMAINS))
diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.py
index c49a2eb04..de5e01d51 100644
--- a/cyberdrop_dl/crawlers/kemono.py
+++ b/cyberdrop_dl/crawlers/kemono.py
@@ -1,16 +1,534 @@
from __future__ import annotations
-from typing import ClassVar
+import asyncio
+import functools
+import itertools
+import re
+from collections import defaultdict
+from datetime import datetime # noqa: TC003
+from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Concatenate, Literal, NamedTuple, NotRequired, ParamSpec
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
+from pydantic import AliasChoices, BeforeValidator, Field
+from typing_extensions import TypedDict # Import from typing is not compatible with pydantic
-from ._kemono_base import KemonoBaseCrawler
+from cyberdrop_dl.crawlers.crawler import Crawler, auto_task_id
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError
+from cyberdrop_dl.models import AliasModel
+from cyberdrop_dl.models.validators import falsy_as_none
+from cyberdrop_dl.utils import css
+from cyberdrop_dl.utils.dates import to_timestamp
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
+
+ from aiohttp_client_cache.response import AnyResponse
+ from bs4 import BeautifulSoup
+
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+ _P = ParamSpec("_P")
+
+
+_DEFAULT_PAGE_SIZE = 50
+_DISCORD_CHANNEL_PAGE_SIZE = 150
+_POST_SELECTOR = "article.post-card a"
+_find_http_urls = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))").finditer
+
+
+class PostSelectors:
+ FILES = "div[class*=__files]"
+ CONTENT = "div[class*=__content]"
+ ATTACHMENTS = "a[class*=__attachment-link]"
+ IMAGES = "div[class=fileThumb]"
+ VIDEOS = "video[class*=__video] source"
+ ALL_CONTENT = f"{FILES}, {CONTENT}"
+
+ DATE_PUBLISHED = "div[class*=__published] time[class=timestamp]"
+ DATE_ADDED = "div[class*=__added]"
+ DATE = f"{DATE_PUBLISHED}, {DATE_ADDED}"
+
+ TITLE = "h1[class*=__title]"
+ USERNAME = "a[class*=__user-name]"
+
+
+class DiscordChannel(NamedTuple):
+ name: str
+ id: str
+
+
+class DiscordServer(NamedTuple):
+ name: str
+ id: str
+ channels: tuple[DiscordChannel, ...]
+
+
+class User(NamedTuple):
+ service: str
+ id: str
+
+
+class File(TypedDict):
+ name: NotRequired[str] # Sometimes present
+ path: str
+ server: NotRequired[str] # Sometimes present in attachments
+
+
+FileOrNone = Annotated[File | None, BeforeValidator(falsy_as_none)]
+
+
+class Post(AliasModel):
+ id: str
+ content: str = ""
+ file: FileOrNone = None
+ attachments: list[File] = [] # noqa: RUF012
+ published_or_added: datetime | None = Field(None, validation_alias=AliasChoices("published", "added"))
+ date: int | None = None
+
+ # `Any` to skip validation, but these are `yarl.URL`. We generate them internally so no validation is needed
+ soup_attachments: list[Any] = [] # noqa: RUF012
+
+ def model_post_init(self, *_) -> None:
+ if self.published_or_added:
+ self.date = to_timestamp(self.published_or_added)
+
+ @property
+ def all_files(self) -> Generator[File]:
+ if self.file:
+ yield self.file
+ yield from self.attachments
+
+
+class UserPost(Post):
+ service: str
+ user_id: str = Field(validation_alias="user")
+ title: str
+
+ @property
+ def user(self) -> User:
+ return User(self.service, self.user_id)
+
+ @property
+ def web_path_qs(self) -> str:
+ return f"{self.service}/user/{self.user_id}/post/{self.id}"
+
+
+class DiscordPost(Post):
+ server_id: str = Field(validation_alias="server")
+ channel_id: str = Field(validation_alias="channel")
+
+ @property
+ def web_path_qs(self) -> str:
+ return f"discord/server/{self.server_id}/{self.channel_id}#{self.id}"
+
+
+class PartialUserPost(NamedTuple):
+ """A simplified version of Post that we can built from a soup, for sites that do not have an API.
+
+ Cons: We need to make a request for every individual post"""
+
+ title: str = ""
+ content: str = ""
+ user_name: str = ""
+ date: str | None = None
+
+ @staticmethod
+ def from_soup(soup: BeautifulSoup) -> PartialUserPost:
+ params = {}
+ selectors = (PostSelectors.TITLE, PostSelectors.ALL_CONTENT, PostSelectors.USERNAME, PostSelectors.DATE)
+ for name, selector in zip(PartialUserPost._fields, selectors, strict=True):
+ if tag := soup.select_one(selector):
+ params[name] = tag.get_text().strip()
+
+ return PartialUserPost(**params)
+
+
+def fallback_if_no_api(
+ func: Callable[Concatenate[KemonoBaseCrawler, _P], Coroutine[None, None, None]],
+) -> Callable[Concatenate[KemonoBaseCrawler, _P], Coroutine[None, None, None]]:
+ """Calls a fallback method is the current instance does not define an API"""
+
+ @functools.wraps(func)
+ async def wrapper(self: KemonoBaseCrawler, *args: _P.args, **kwargs: _P.kwargs) -> None:
+ if getattr(self, "API_ENTRYPOINT", None):
+ return await func(self, *args, **kwargs)
+
+ if fallback_func := getattr(self, f"{func.__name__}_w_no_api", None):
+ return await fallback_func(*args, **kwargs)
+
+ raise ValueError
+
+ return wrapper
+
+
+class KemonoBaseCrawler(Crawler, is_abc=True):
+ SUPPORTED_PATHS: ClassVar[dict[str, str]] = {
+ "Model": "/<service>/user/<user_id>",
+ "Favorites": "/favorites/<user_id>",
+ "Search": "/search?q=...",
+ "Individual Post": "/<service>/user/<user_id>/post/<post_id>",
+ "Direct links": "/(data|thumbnail)/...",
+ }
+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {title}"
+ API_ENTRYPOINT: ClassVar[AbsoluteHttpURL]
+ SERVICES: ClassVar[tuple[str, ...]] = ()
+
+ def __init_subclass__(cls, **kwargs) -> None:
+ super().__init_subclass__(**kwargs)
+ Crawler._assert_fields_overrides(cls, "SERVICES")
+
+ def __post_init__(self) -> None:
+ self._user_names: dict[User, str] = {}
+ self.__known_discord_servers: dict[str, DiscordServer] = {}
+ self.__known_attachment_servers: dict[str, str] = {}
+ self.__discord_servers_locks: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
+
+ @property
+ def session_cookie(self) -> str:
+ return ""
+
+ async def async_startup(self) -> None:
+ def check_kemono_page(response: AnyResponse) -> bool:
+ if any(x in response.url.parts for x in self.SERVICES):
+ return False
+ if "discord/channel" in response.url.path:
+ return False
+ return True
+
+ self.register_cache_filter(self.PRIMARY_URL, check_kemono_page)
+ if getattr(self, "API_ENTRYPOINT", None):
+ await self._get_usernames(self.API_ENTRYPOINT / "creators")
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case [service, "user", _, "post", _] if service in self.SERVICES:
+ return await self.post(scrape_item)
+ case [service, "user", _] if service in self.SERVICES:
+ return await self.profile(scrape_item)
+ case ["favorites", _] if (type_ := scrape_item.url.query.get("type")) in ("post", "artist"):
+ return await self.favorites(scrape_item, type_)
+ case ["posts"] if search_query := scrape_item.url.query.get("q"):
+ return await self.search(scrape_item, search_query)
+ case ["discord", "server", server_id]:
+ return await self.discord_server(scrape_item, server_id)
+ case ["discord", "server", _, channel_id]:
+ return await self.discord_channel(scrape_item, channel_id)
+ case ["thumbnail" | "thumbnails" | "data", _, *_]:
+ return await self.handle_direct_link(scrape_item)
+ case _:
+ raise ValueError
+
+ @fallback_if_no_api
+ @error_handling_wrapper
+ async def search(self, scrape_item: ScrapeItem, query: str) -> None:
+ api_url = self.__make_api_url_w_offset(scrape_item.url, "posts")
+ title = self.create_title(f"{query} [search]")
+ scrape_item.setup_as_profile(title)
+ await self.__iter_user_posts(scrape_item, api_url)
+
+ @fallback_if_no_api
+ @error_handling_wrapper
+ async def profile(self, scrape_item: ScrapeItem) -> None:
+ path = (scrape_item.url / "posts").path
+ api_url = self.__make_api_url_w_offset(scrape_item.url, path)
+ scrape_item.setup_as_profile("")
+ await self.__iter_user_posts(scrape_item, api_url)
+
+ @fallback_if_no_api
+ @error_handling_wrapper
+ async def discord_server(self, scrape_item: ScrapeItem, server_id: str) -> None:
+ scrape_item.setup_as_forum("")
+ server = await self.__get_discord_server(server_id)
+ for channel in server.channels:
+ url = self.PRIMARY_URL / "discord/server" / server_id / channel.id
+ new_scrape_item = scrape_item.create_child(url)
+ self.create_task(self.run(new_scrape_item))
+ scrape_item.add_children()
+
+ @fallback_if_no_api
+ @error_handling_wrapper
+ async def discord_channel(self, scrape_item: ScrapeItem, channel_id: str) -> None:
+ scrape_item.setup_as_profile("")
+ api_url = self.__make_api_url_w_offset(scrape_item.url, f"discord/channel/{channel_id}")
+ async for posts in self._pager(api_url, step_size=_DISCORD_CHANNEL_PAGE_SIZE):
+ if not isinstance(posts, list):
+ error_msg = (
+ f"[{self.NAME}] Invalid API response for Discord channel '{channel_id}' posts (URL: {api_url}). "
+ f"Expected a list, but got type {type(posts).__name__}. "
+ f"Response data (truncated): {str(posts)[:200]}"
+ )
+ raise ScrapeError(422, error_msg)
+
+ for post_data in posts:
+ if not isinstance(post_data, dict):
+ error_msg = (
+ f"[{self.NAME}] Invalid post data type in list for "
+ f"Discord channel '{channel_id}' (URL template: {api_url}). "
+ f"Expected a dict for post data, but got type {type(post_data).__name__}. "
+ f"Post data (truncated): {str(post_data)[:200]}"
+ )
+ raise ScrapeError(422, error_msg)
+
+ post = DiscordPost.model_validate(post_data)
+ post_web_url = self.parse_url(post.web_path_qs)
+ new_scrape_item = scrape_item.create_child(post_web_url)
+ self.create_task(self._handle_discord_post_task(new_scrape_item, post))
+ scrape_item.add_children()
+
+ if len(posts) < _DISCORD_CHANNEL_PAGE_SIZE:
+ break
+
+ @fallback_if_no_api
+ @error_handling_wrapper
+ async def post(self, scrape_item: ScrapeItem) -> None:
+ api_url = self.__make_api_url_w_offset(scrape_item.url)
+ json_resp: dict[str, Any] = await self.__api_request(api_url)
+ post = UserPost.model_validate(json_resp["post"])
+ self._register_attachments_servers(json_resp["attachments"])
+ self._handle_user_post(scrape_item, post)
+
+ @fallback_if_no_api
+ @error_handling_wrapper
+ async def favorites(self, scrape_item: ScrapeItem, type_: Literal["post", "artist"]) -> None:
+ if not self.session_cookie:
+ msg = "No session cookie found in the config file, cannot scrape favorites"
+ raise ScrapeError(401, msg)
+
+ title = f"My favorite {type_}s"
+ scrape_item.setup_as_profile(self.create_title(title))
+ self.update_cookies({"session": self.session_cookie})
+ query_url = (self.API_ENTRYPOINT / "account/favorites").with_query(type=type_)
+ resp: list[dict[str, Any]] = await self.__api_request(query_url)
+ self.update_cookies({"session": ""})
+
+ for item in resp:
+ url = self.PRIMARY_URL / item["service"] / "user" / item["user"]
+ if type_ == "post":
+ url = url / "post" / item["id"]
+
+ new_scrape_item = scrape_item.create_child(url)
+ self.create_task(self.run(new_scrape_item))
+
+ @error_handling_wrapper
+ async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:
+ scrape_item.url = _thumbnail_to_src(scrape_item.url)
+ link = _thumbnail_to_src(url or scrape_item.url)
+ try:
+ filename, ext = self.get_filename_and_ext(link.query.get("f") or link.name)
+ except NoExtensionError:
+ # Some patreon URLs have another URL as the filename:
+ # ex: https://kemono.su/data/7a...27ad7e40bd.jpg?f=https://www.patreon.com/media-u/Z0F..00672794_
+ filename, ext = self.get_filename_and_ext(link.name)
+ await self.handle_file(link, scrape_item, link.name, ext, custom_filename=filename)
+
+ # ~~~~~~~~ INTERNAL METHODS, not expected to be overriden, but could be ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ def _register_attachments_servers(self, attachments: list[File]) -> None:
+ for attach in attachments:
+ server = attach.get("server")
+ if not server:
+ continue
+
+ path = attach["path"]
+ if previous_server := self.__known_attachment_servers.get(path):
+ if previous_server != server:
+ msg = (
+ f"[{self.NAME}] {path} found with multiple " #
+ f"different servers: {server = } {previous_server = } "
+ )
+ self.log(msg, 30)
+ continue
+ self.__known_attachment_servers[path] = server
+
+ def _handle_user_post(self, scrape_item: ScrapeItem, post: UserPost) -> None:
+ user_name = self._user_names[post.user]
+ title = self.create_title(user_name, post.user_id)
+ scrape_item.setup_as_album(title, album_id=post.user_id)
+ scrape_item.possible_datetime = post.date
+ post_title = self.create_separate_post_title(post.title, post.id, post.date)
+ scrape_item.add_to_parent_title(post_title)
+ self.__handle_post(scrape_item, post)
+
+ async def _handle_discord_post(self, scrape_item: ScrapeItem, post: DiscordPost) -> None:
+ server = await self.__get_discord_server(post.server_id)
+ title = self.create_title(f"{server.name} [discord]", server.id)
+ channel_name = next(c.name for c in server.channels if c.id == post.channel_id)
+ scrape_item.setup_as_album(title, album_id=server.id)
+ scrape_item.possible_datetime = post.date
+ scrape_item.add_to_parent_title(f"#{channel_name}")
+ post_title = self.create_separate_post_title(None, post.id, post.date)
+ scrape_item.add_to_parent_title(post_title)
+ self.__handle_post(scrape_item, post)
+
+ _handle_discord_post_task = auto_task_id(_handle_discord_post)
+
+ def _handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:
+ """Gets links out of content in post and sends them to a new crawler."""
+ if not post.content:
+ return
+
+ for link in self.__parse_content_urls(post):
+ new_scrape_item = scrape_item.create_child(link)
+ self.handle_external_links(new_scrape_item)
+ scrape_item.add_children()
+
+ @error_handling_wrapper
+ async def _get_usernames(self, api_url: AbsoluteHttpURL) -> None:
+ try:
+ json_resp: list[dict[str, Any]] = await self.__api_request(api_url)
+ self._user_names = {User(u["service"], u.get("user_id") or u["id"]): u["name"] for u in json_resp}
+ except Exception as e:
+ msg = f"Unable to get list of creators from {self.NAME}. Crawler has been disabled"
+ self.disabled = True
+ raise ScrapeError(503, msg) from e
+
+ """~~~~~~~~ PRIVATE METHODS, should never be overriden ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
+
+ def __parse_content_urls(self, post: Post) -> Generator[AbsoluteHttpURL]:
+ seen: set[str] = set()
+ for match in _find_http_urls(post.content):
+ if (link := match.group().replace(".md.", ".")) not in seen:
+ seen.add(link)
+ try:
+ url = self.parse_url(link)
+ except Exception:
+ pass
+ else:
+ if self.DOMAIN not in url.host:
+ yield url
+
+ def __handle_post(self, scrape_item: ScrapeItem, post: Post) -> None:
+ for file in post.all_files:
+ file_url = self.__make_file_url(file)
+ self.create_task(self.handle_direct_link(scrape_item, file_url))
+ scrape_item.add_children()
+
+ for file_url in post.soup_attachments:
+ self.create_task(self.handle_direct_link(scrape_item, file_url))
+ scrape_item.add_children()
+
+ self._handle_post_content(scrape_item, post)
+
+ def __make_file_url(self, file: File) -> AbsoluteHttpURL:
+ path = file["path"]
+ server = self.__known_attachment_servers.get(path) or ""
+ url = self.parse_url(server + f"/data{path}")
+ return url.with_query(f=file.get("name") or url.name)
+
+ def __make_api_url_w_offset(self, web_url: AbsoluteHttpURL, path: str | None = None) -> AbsoluteHttpURL:
+ api_url = self.API_ENTRYPOINT / (path or web_url.path).removeprefix("/")
+ offset = int(web_url.query.get("o", 0))
+ if query := web_url.query.get("q"):
+ return api_url.update_query(o=offset, q=query)
+ return api_url.update_query(o=offset)
+
+ async def __get_discord_server(self, server_id: str) -> DiscordServer:
+ """Get discord server information, making new API calls if needed."""
+ async with self.__discord_servers_locks[server_id]:
+ if server := self.__known_discord_servers.get(server_id):
+ return server
+
+ server_api_url = self.API_ENTRYPOINT / "discord/user" / server_id / "profile"
+ server_profile: dict[str, Any] = await self.__api_request(server_api_url)
+ name = server_profile.get("name") or f"Discord Server {server_id}"
+ channels_api_url = self.API_ENTRYPOINT / "discord/channel/lookup" / server_id
+ channels_resp: list[dict] = await self.__api_request(channels_api_url)
+ channels = tuple(DiscordChannel(channel["name"], channel["id"]) for channel in channels_resp)
+ self.__known_discord_servers[server_id] = server = DiscordServer(name, server_id, channels)
+ return server
+
+ async def __iter_user_posts(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL) -> None:
+ async for json_resp in self._pager(url):
+ # From search results
+ if isinstance(json_resp, dict):
+ posts = json_resp.get("posts", [])
+
+ # From profile
+ elif isinstance(json_resp, list):
+ posts: list[dict[str, Any]] = json_resp
+
+ else:
+ raise ScrapeError(422)
+
+ for post in (UserPost.model_validate(entry) for entry in posts):
+ post_web_url = self.parse_url(post.web_path_qs)
+ new_scrape_item = scrape_item.create_child(post_web_url)
+ self._handle_user_post(new_scrape_item, post)
+ scrape_item.add_children()
+
+ if len(posts) < _DEFAULT_PAGE_SIZE:
+ break
+
+ async def _pager(self, url: AbsoluteHttpURL, step_size: int | None = None) -> AsyncGenerator[Any]:
+ """Yields JSON responses from API calls, or soup for web page calls, with configurable increments."""
+ current_step_size = step_size or _DEFAULT_PAGE_SIZE
+ init_offset = int(url.query.get("o") or 0)
+
+ request = self.__api_request if "api" in url.parts else self.request_soup
+ for current_offset in itertools.count(init_offset, current_step_size):
+ yield await request(url.update_query(o=current_offset))
+
+ async def __api_request(self, url: AbsoluteHttpURL) -> Any:
+ """Get JSON from the API, with a custom Accept header."""
+
+ # When using the 'text/css' header, the response is missing the charset header
+ # and charset detection may return a random codec if the body has non english chars, so we force utf-8
+ async with self.request(url, headers={"Accept": "text/css"}) as resp:
+ return await resp.json(encoding="utf-8", content_type=False)
+
+ # ~~~~~~~~~~ NO API METHODS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ @error_handling_wrapper
+ async def profile_w_no_api(self, scrape_item: ScrapeItem) -> None:
+ scrape_item.setup_as_profile("")
+ soup: BeautifulSoup
+ async for soup in self._pager(scrape_item.url):
+ n_posts = 0
+
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _POST_SELECTOR):
+ n_posts += 1
+ self.create_task(self.post_w_no_api_task(new_scrape_item))
+
+ if n_posts < _DEFAULT_PAGE_SIZE:
+ break
+
+ @error_handling_wrapper
+ async def post_w_no_api(self, scrape_item: ScrapeItem) -> None:
+ soup = await self.request_soup(scrape_item.url)
+ service, _, user_id, _, post_id = scrape_item.url.parts[1:6]
+ partial_post = PartialUserPost.from_soup(soup)
+ if not partial_post.title or not partial_post.user_name:
+ raise ScrapeError(422)
+
+ def files():
+ for selector in (
+ PostSelectors.VIDEOS,
+ PostSelectors.IMAGES,
+ PostSelectors.ATTACHMENTS,
+ ):
+ for file in soup.select(selector):
+ yield self.parse_url(css.get_attr(file, "href"))
+
+ post = UserPost(
+ user_id=user_id,
+ service=service,
+ id=post_id,
+ title=partial_post.title,
+ content=partial_post.content,
+ published_or_added=partial_post.date, # type: ignore[reportArgumentType]
+ soup_attachments=list(files()),
+ )
+
+ self._handle_user_post(scrape_item, post)
+
+ post_w_no_api_task = auto_task_id(post_w_no_api)
class KemonoCrawler(KemonoBaseCrawler):
SUPPORTED_PATHS: ClassVar = KemonoBaseCrawler.SUPPORTED_PATHS | {
"Discord Server": "/discord/<server_id>",
- "Discord Server Channel": "/discord/server/...#...",
+ "Discord Server Channel": "/discord/server/<server_id>/<channel_id>#...",
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://kemono.cr")
API_ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://kemono.cr/api/v1")
@@ -27,7 +545,13 @@ class KemonoCrawler(KemonoBaseCrawler):
)
OLD_DOMAINS: ClassVar[tuple[str, ...]] = "kemono.party", "kemono.su"
- async def fetch(self, scrape_item: ScrapeItem) -> None:
- if "discord" in scrape_item.url.parts:
- return await self.discord(scrape_item)
- return await super().fetch(scrape_item)
+ @property
+ def session_cookie(self):
+ return self.manager.config_manager.authentication_data.kemono.session
+
+
+def _thumbnail_to_src(og_url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ url = remove_parts(og_url, "thumbnails", "thumbnail").with_query(None)
+ if name := og_url.query.get("f"):
+ return url.with_query(f=name)
+ return url
diff --git a/cyberdrop_dl/crawlers/nekohouse.py b/cyberdrop_dl/crawlers/nekohouse.py
index 758300a09..728fca195 100644
--- a/cyberdrop_dl/crawlers/nekohouse.py
+++ b/cyberdrop_dl/crawlers/nekohouse.py
@@ -1,33 +1,24 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+from typing import ClassVar
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from ._kemono_base import KemonoBaseCrawler
-
-if TYPE_CHECKING:
- from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+from .kemono import KemonoBaseCrawler
class NekohouseCrawler(KemonoBaseCrawler):
SUPPORTED_PATHS: ClassVar[dict[str, str]] = {
- "Model": "/<service>/user/<user>",
- "Individual Post": "/<service>/<user>/post/...",
- "Direct links": "",
+ "Model": "/<service>/user/<user_id>",
+ "Individual Post": "/<service>/user/<user_id>/post/<post_id>",
+ "Direct links": "/(data|thumbnails)/...",
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://nekohouse.su")
DOMAIN: ClassVar[str] = "nekohouse"
SERVICES = "fanbox", "fantia", "fantia_products", "subscribestar", "twitter"
- async def fetch(self, scrape_item: ScrapeItem) -> None:
- if "thumbnails" in scrape_item.url.parts:
- return await self.handle_direct_link(scrape_item)
- if "post" in scrape_item.url.parts:
- return await self.post_w_no_api(scrape_item)
- if any(x in scrape_item.url.parts for x in self.SERVICES):
- return await self.profile_w_no_api(scrape_item)
- if any(x in scrape_item.url.parts for x in ("posts", "discord")):
- raise ValueError
+ async def async_startup(self) -> None:
+ await super().async_startup()
- await self.handle_direct_link(scrape_item)
+ # Only this API endpoint is available
+ await self._get_usernames(self.PRIMARY_URL / "api/creators")
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 3b9927264..b7d16d146 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -181,7 +181,7 @@ class Downloader:
self.update_queued_files()
task_id = self.manager.progress_manager.file_progress.add_task(domain=self.domain, filename=media_item.filename)
media_item.set_task_id(task_id)
- video, audio, subtitles = await self._download_rendition_group(media_item, m3u8_group)
+ video, audio, _ = await self._download_rendition_group(media_item, m3u8_group)
if not audio:
await asyncio.to_thread(video.rename, media_item.complete_file)
else:
diff --git a/tests/crawlers/test_cases/coomer.py b/tests/crawlers/test_cases/coomer.py
new file mode 100644
index 000000000..7ada225ab
--- /dev/null
+++ b/tests/crawlers/test_cases/coomer.py
@@ -0,0 +1,41 @@
+DOMAIN = "coomer"
+TEST_CASES = [
+ (
+ "https://coomer.st/fansly/user/445978322480930816/post/644032890815913985",
+ [
+ {
+ "url": "https://coomer.st/data/03/ef/03ef02264303b2693c2acb714cf0321d3bfca04b85511dc4f66887ad134cb69e.jpg?f=644032704089694208.jpeg",
+ "filename": "644032704089694208.jpeg",
+ "referer": "https://coomer.st/fansly/user/445978322480930816/post/644032890815913985",
+ "download_folder": r"re:zombonn \(Coomer\)",
+ "album_id": "445978322480930816",
+ "datetime": 1715061766,
+ }
+ ],
+ ),
+ (
+ "https://coomer.st/onlyfans/user/lostfaefoundskates/post/729202326",
+ [
+ {
+ "url": "https://coomer.st/data/fe/af/feafbd06f78da02309df109c6ed4b83dd877cc256ecfc41212d5adf8673166a6.jpg?f=3840x2880_ccd7f56947c484db31c59421343c71cd.jpg",
+ "filename": "3840x2880_ccd7f56947c484db31c59421343c71cd.jpg",
+ "referer": "https://coomer.st/onlyfans/user/lostfaefoundskates/post/729202326",
+ "download_folder": r"re:lostfaefoundskates \(Coomer\)",
+ "album_id": "lostfaefoundskates",
+ "datetime": 1694414991,
+ }
+ ],
+ ),
+ (
+ "https://coomer.st/data/4b/de/4bde70ea2dd6284938aa90af959f78c4f3e24fbf6b06ed842f0356824fe9bef0.jpg?f=3024x4032_c0ef341d7e60ceed82a906b8062806cb.jpg",
+ [
+ {
+ "url": "https://coomer.st/data/4b/de/4bde70ea2dd6284938aa90af959f78c4f3e24fbf6b06ed842f0356824fe9bef0.jpg?f=3024x4032_c0ef341d7e60ceed82a906b8062806cb.jpg",
+ "filename": "3024x4032_c0ef341d7e60ceed82a906b8062806cb.jpg",
+ "referer": "https://coomer.st/data/4b/de/4bde70ea2dd6284938aa90af959f78c4f3e24fbf6b06ed842f0356824fe9bef0.jpg?f=3024x4032_c0ef341d7e60ceed82a906b8062806cb.jpg",
+ "album_id": None,
+ "datetime": None,
+ },
+ ],
+ ),
+]
diff --git a/tests/crawlers/test_cases/kemono.py b/tests/crawlers/test_cases/kemono.py
old mode 100644
new mode 100755
index 673d93993..7f0d61b39
--- a/tests/crawlers/test_cases/kemono.py
+++ b/tests/crawlers/test_cases/kemono.py
@@ -1,15 +1,65 @@
DOMAIN = "kemono"
TEST_CASES = [
(
- "https://kemono.su/patreon/user/27174868/post/133667042",
+ "https://kemono.cr/patreon/user/27174868/post/133667042",
[
{
- "url": "https://kemono.su/data/0d/c3/0dc3ecec556918b882674c5dd63f8fa1867867157ce1ca675b0bf79e6b73fb04.jpg?f=Birthday_Kim.jpg",
+ "url": "https://kemono.cr/data/0d/c3/0dc3ecec556918b882674c5dd63f8fa1867867157ce1ca675b0bf79e6b73fb04.jpg?f=Birthday_Kim.jpg",
"filename": "Birthday_Kim.jpg",
- "referer": "https://kemono.su/patreon/user/27174868/post/133667042",
+ "referer": "https://kemono.cr/patreon/user/27174868/post/133667042",
"album_id": "27174868",
"datetime": 1752004344,
}
],
- )
+ ),
+ (
+ "https://kemono.cr/data/49/83/49830d8d281ad9a508f88b081b3a112aa4011a1af3bb2539f88f4e635edf98a1.png?f=june_Schedule.png",
+ [
+ {
+ "url": "https://kemono.cr/data/49/83/49830d8d281ad9a508f88b081b3a112aa4011a1af3bb2539f88f4e635edf98a1.png?f=june_Schedule.png",
+ "filename": "june_Schedule.png",
+ "referer": "https://kemono.cr/data/49/83/49830d8d281ad9a508f88b081b3a112aa4011a1af3bb2539f88f4e635edf98a1.png?f=june_Schedule.png",
+ "album_id": None,
+ "datetime": None,
+ },
+ ],
+ ),
+ (
+ "https://kemono.cr/discord/server/794379082535927828/921981855706796032",
+ [
+ {
+ "url": "https://kemono.cr/data/49/83/49830d8d281ad9a508f88b081b3a112aa4011a1af3bb2539f88f4e635edf98a1.png?f=june_Schedule.png",
+ "filename": "june_Schedule.png",
+ "referer": "https://kemono.cr/discord/server/794379082535927828/921981855706796032#1257906262025175090",
+ "download_folder": r"re:Schpicy-s server \[discord\] \(Kemono\)\/\#schedule",
+ "album_id": "794379082535927828",
+ "datetime": 1719996623,
+ },
+ ],
+ 3,
+ ),
+ (
+ "https://kemono.cr/patreon/user/16573132",
+ [
+ {
+ "url": "https://kemono.cr/data/01/1e/011e7a27eca93eba99ab223553d6079495a55c87106fa0e36efb8810048da92f.jpg?f=httpswww.patreon.commedia-uZ0FBQUFBQmdWWnhiV0xoYnpvV0Nnd0ZvMnM5UnkyTVF1LXd4MEpNYmVRWGNMNEZhUE1wWWJreVBqS1F2UzB3R2lYaXpDaXBWUy14M2l1WGtITWJBX0Q4ZDRoQzBtOG5nd3JIYThERTRWNDIxeUY4RHBXaERxWkRJNS1xRXRPS2lFM3pLSEVTSW5oNVRzeU9yc21SMmo5MkJrYXVXVzFCVUtRPT0",
+ "filename": "011e7a27eca93eba99ab223553d6079495a55c87106fa0e36efb8810048da92f.jpg",
+ "referer": "https://kemono.cr/patreon/user/16573132/post/42836761",
+ "download_folder": r"re:EUNSONGS \(Kemono\)",
+ "album_id": "16573132",
+ "datetime": 1602954002,
+ }
+ ],
+ 212,
+ ),
+ (
+ "https://kemono.cr/posts?q=dandadan%20s2%20episode%208%20reaction",
+ [
+ {
+ "url": "ANY",
+ "download_folder": r"re:dandadan s2 episode 8 reaction \[search\] \(Kemono\)",
+ }
+ ],
+ 43,
+ ),
]
diff --git a/tests/crawlers/test_cases/nekohouse.py b/tests/crawlers/test_cases/nekohouse.py
new file mode 100644
index 000000000..4f32a8bbd
--- /dev/null
+++ b/tests/crawlers/test_cases/nekohouse.py
@@ -0,0 +1,56 @@
+DOMAIN = "nekohouse"
+TEST_CASES = [
+ (
+ "https://nekohouse.su/fanbox/user/705370/post/9605906",
+ [
+ {
+ "url": "https://nekohouse.su/data/33/7f/337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg",
+ "filename": "337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg",
+ "referer": "https://nekohouse.su/fanbox/user/705370/post/9605906",
+ "download_folder": r"re:shiratamaco \(Nekohouse\)",
+ "album_id": "705370",
+ "datetime": 1743063817,
+ },
+ {
+ "url": "https://nekohouse.su/data/ec/7b/ec7bf13520488ea9ed19e766eeac6033547e69eca88e395f12c9e503feaa71a5.jpg",
+ "filename": "ec7bf13520488ea9ed19e766eeac6033547e69eca88e395f12c9e503feaa71a5.jpg",
+ "referer": "https://nekohouse.su/fanbox/user/705370/post/9605906",
+ "album_id": "705370",
+ "datetime": 1743063817,
+ },
+ {
+ "url": "https://nekohouse.su/data/fa/66/fa66c78e0fefaa748baeee23dafc5392236b3a194aa0f2b3040dced30a617b7f.jpg",
+ "filename": "fa66c78e0fefaa748baeee23dafc5392236b3a194aa0f2b3040dced30a617b7f.jpg",
+ "referer": "https://nekohouse.su/fanbox/user/705370/post/9605906",
+ "album_id": "705370",
+ "datetime": 1743063817,
+ },
+ ],
+ ),
+ (
+ "https://nekohouse.su/thumbnail/data/33/7f/337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg?type=preview",
+ [
+ {
+ "url": "https://nekohouse.su/data/33/7f/337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg",
+ "filename": "337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg",
+ "referer": "https://nekohouse.su/data/33/7f/337f1f97e0414358092fe9de88046614e35aab417d030da820e4816e3cab3e42.jpg",
+ "album_id": None,
+ "datetime": None,
+ },
+ ],
+ ),
+ (
+ "https://nekohouse.su/fanbox/user/92778036",
+ [
+ {
+ "url": "https://nekohouse.su/data/01/44/014495e8f29a8a31ac008ff0038392f738a5237eff36c06a29b35f60d9ef470e.png",
+ "filename": "014495e8f29a8a31ac008ff0038392f738a5237eff36c06a29b35f60d9ef470e.png",
+ "referer": "https://nekohouse.su/fanbox/user/92778036/post/9570757",
+ "download_folder": r"re:redkingtaraba \(Nekohouse\)",
+ "album_id": "92778036",
+ "datetime": 1742473273,
+ },
+ ],
+ 549,
+ ),
+]