Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add coomerfans support (#1771)

NTFSvolume87baec7930dc
diff --git a/cyberdrop_dl/crawlers/coomerfans.py b/cyberdrop_dl/crawlers/coomerfans.py
new file mode 100644
index 000000000..d6901e25c
--- /dev/null
+++ b/cyberdrop_dl/crawlers/coomerfans.py
@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import dataclasses
+import itertools
+import re
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
+
+ from bs4 import BeautifulSoup
+
+ from cyberdrop_dl.url_objects import ScrapeItem
+
+_find_http_urls = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))").finditer
+
+
[email protected](slots=True)
+class Post:
+ date: str
+ model_name: str
+ content: str
+ title: str
+ images: tuple[str, ...]
+ videos: tuple[str, ...]
+
+
+class CoomerFansCrawler(Crawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Post": "/p/<post_id>/...",
+ "User": "/u/<user_id>/...",
+ "**NOTE**": "`--ignore-coomer-post-content` affects this crawler. All other kemono config options are ignored",
+ }
+
+ DOMAIN: ClassVar[str] = "coomerfans"
+ FOLDER_DOMAIN: ClassVar[str] = "CoomerFans"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://coomerfans.com")
+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {title}"
+ NEXT_PAGE_SELECTOR: ClassVar[str] = ".pagination a.next"
+
+ @classmethod
+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ url = super().transform_url(url)
+ match url.parts[1:]:
+ case ["p", _, *rest] | ["u", _, _, *rest] if not rest:
+ return url / ""
+ case _:
+ return url
+
+ @property
+ def ignore_content(self) -> bool:
+ return self.manager.config.settings.ignore_options.ignore_coomer_post_content
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["p", post_id, *_]:
+ return await self.post(scrape_item, post_id)
+ case ["u", _service, _user_id, *_]:
+ return await self.profile(scrape_item)
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def post(self, scrape_item: ScrapeItem, post_id: str) -> None:
+ soup = await self.request_soup(scrape_item.url)
+ post = _parse_post(soup)
+ scrape_item.setup_as_album(self.create_title(post.model_name))
+ scrape_item.uploaded_at = date = self.parse_iso_date(post.date)
+ post_title = self.create_separate_post_title(post.title, post_id, date)
+ scrape_item.add_to_parent_title(post_title)
+ self.create_task(self.write_metadata(scrape_item, f"post_{post_id}", post))
+ self._post(scrape_item, post)
+
+ def _post(self, scrape_item: ScrapeItem, post: Post) -> None:
+ seen: set[str] = set()
+ for url in itertools.chain(post.images, post.videos):
+ if url not in seen:
+ seen.add(url)
+ self.create_task(self.direct_file(scrape_item, self.parse_url(url)))
+ scrape_item.add_children()
+
+ if not post.content or self.ignore_content:
+ return
+
+ for url in self.__parse_content_urls(post):
+ self.handle_external_links(scrape_item.create_child(url))
+ scrape_item.add_children()
+
+ def __parse_content_urls(self, post: Post) -> Generator[AbsoluteHttpURL]:
+ seen: set[str] = set()
+ for match in _find_http_urls(post.content):
+ if (link := match.group().replace(".md.", ".")) not in seen:
+ seen.add(link)
+ try:
+ url = self.parse_url(link)
+ except Exception:
+ pass
+ else:
+ if self.DOMAIN not in url.host:
+ yield url
+
+ @error_handling_wrapper
+ async def profile(self, scrape_item: ScrapeItem) -> None:
+ async for soup in self.web_pager(scrape_item.url):
+ for _, child in self.iter_children(scrape_item, soup, "a.view-post[href]"):
+ self.create_task(self.run(child))
+
+
+def _parse_post(soup: BeautifulSoup) -> Post:
+ main = css.select(soup, "main.content")
+ body = css.select(main, ".post-body")
+
+ def get(attr: str) -> str:
+ return css.select_text(main, attr)
+
+ return Post(
+ title=get("h1"),
+ model_name=get(".model-name"),
+ content=get(".post-date + p"),
+ date=extr_text(get(".post-date"), "Added ", " +"),
+ images=tuple(css.iselect(body, "img", "src")),
+ videos=tuple(css.iselect(body, "source", "src")),
+ )
diff --git a/tests/crawlers/test_cases/coomerfans.py b/tests/crawlers/test_cases/coomerfans.py
new file mode 100644
index 000000000..18947c090
--- /dev/null
+++ b/tests/crawlers/test_cases/coomerfans.py
@@ -0,0 +1,171 @@
+DOMAIN = "coomerfans"
+TEST_CASES = [
+ (
+ "https://coomerfans.com/p/63797958/315883/onlyfans",
+ [
+ {
+ "url": "https://img1.coomerfans.com/storage/3/xt/qi/2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",
+ "filename": "2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",
+ "debrid_link": None,
+ "original_filename": "2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",
+ "referer": "https://coomerfans.com/p/63797958/315883/onlyfans",
+ "uploaded_at": 1738836595,
+ "download_folder": "re:carolynnmoon (CoomerFans)",
+ }
+ ],
+ ),
+ (
+ "https://coomerfans.com/p/63797958",
+ [
+ {
+ "url": "https://img1.coomerfans.com/storage/3/xt/qi/2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",
+ "filename": "2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",
+ "debrid_link": None,
+ "original_filename": "2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",
+ "referer": "re:https://coomerfans.com/p/63797958",
+ "uploaded_at": 1738836595,
+ "download_folder": "re:carolynnmoon (CoomerFans)",
+ }
+ ],
+ ),
+ (
+ "https://coomerfans.com/p/73779832/338170/onlyfans",
+ [
+ {
+ "url": "https://img1.coomerfans.com/storage/7/zn/iy/2892ac-01961036-199f-7926-873a-68444ae64ee8.jpg",
+ "filename": "2892ac-01961036-199f-7926-873a-68444ae64ee8.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-199f-7926-873a-68444ae64ee8.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/10/sk/di/2892ac-01961036-19a0-72bc-997c-faa5b57afe74.jpg",
+ "filename": "2892ac-01961036-19a0-72bc-997c-faa5b57afe74.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19a0-72bc-997c-faa5b57afe74.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/4/xb/qs/2892ac-01961036-19a0-730c-8c58-5a8378349587.jpg",
+ "filename": "2892ac-01961036-19a0-730c-8c58-5a8378349587.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19a0-730c-8c58-5a8378349587.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/9/it/xf/2892ac-01961036-19a0-7424-ab8c-be28c49d235e.jpg",
+ "filename": "2892ac-01961036-19a0-7424-ab8c-be28c49d235e.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19a0-7424-ab8c-be28c49d235e.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/2/yz/yc/2892ac-01961036-19a0-75b3-bed0-17b813e28931.jpg",
+ "filename": "2892ac-01961036-19a0-75b3-bed0-17b813e28931.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19a0-75b3-bed0-17b813e28931.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/12/hm/hd/2892ac-01961036-19a0-7bd9-84a3-75a651ebfc3d.jpg",
+ "filename": "2892ac-01961036-19a0-7bd9-84a3-75a651ebfc3d.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19a0-7bd9-84a3-75a651ebfc3d.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/1/lo/vd/2892ac-01961036-19a0-7bd5-9180-146ca7ac6b50.jpg",
+ "filename": "2892ac-01961036-19a0-7bd5-9180-146ca7ac6b50.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19a0-7bd5-9180-146ca7ac6b50.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/2/sg/wb/2892ac-01961036-19a3-7d4e-854c-07a57e9c2975.jpg",
+ "filename": "2892ac-01961036-19a3-7d4e-854c-07a57e9c2975.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19a3-7d4e-854c-07a57e9c2975.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/7/yh/zv/2892ac-01961036-19ab-7b46-bd88-ce9541d9e8f5.jpg",
+ "filename": "2892ac-01961036-19ab-7b46-bd88-ce9541d9e8f5.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19ab-7b46-bd88-ce9541d9e8f5.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/7/sg/ot/2892ac-01961036-19bb-76ff-8a22-5c2e6aef03b2.jpg",
+ "filename": "2892ac-01961036-19bb-76ff-8a22-5c2e6aef03b2.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19bb-76ff-8a22-5c2e6aef03b2.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/4/ns/ds/2892ac-01961036-19bd-741e-8ead-503329369493.jpg",
+ "filename": "2892ac-01961036-19bd-741e-8ead-503329369493.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19bd-741e-8ead-503329369493.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/4/cf/mb/2892ac-01961036-19bd-7897-99d2-95d1d004874b.jpg",
+ "filename": "2892ac-01961036-19bd-7897-99d2-95d1d004874b.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19bd-7897-99d2-95d1d004874b.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/7/on/zf/2892ac-01961036-19d3-7e4b-a226-b57dcc1e7581.jpg",
+ "filename": "2892ac-01961036-19d3-7e4b-a226-b57dcc1e7581.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-19d3-7e4b-a226-b57dcc1e7581.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/6/gq/yr/2892ac-01961036-1a3b-73fb-9c95-125499caa2d6.jpg",
+ "filename": "2892ac-01961036-1a3b-73fb-9c95-125499caa2d6.jpg",
+ "debrid_link": None,
+ "original_filename": "2892ac-01961036-1a3b-73fb-9c95-125499caa2d6.jpg",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ {
+ "url": "https://img1.coomerfans.com/storage/7/fr/ob/2892ac-01970401-3ad7-73d5-bbec-6426c3edb74b.mp4",
+ "filename": "2892ac-01970401-3ad7-73d5-bbec-6426c3edb74b.mp4",
+ "debrid_link": None,
+ "original_filename": "2892ac-01970401-3ad7-73d5-bbec-6426c3edb74b.mp4",
+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",
+ "uploaded_at": 1717563216,
+ "download_folder": "re:belledelphine (CoomerFans)",
+ },
+ ],
+ ),
+]