Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add coomerfans support (#1771)
NTFSvolume87baec7930dc
diff --git a/cyberdrop_dl/crawlers/coomerfans.py b/cyberdrop_dl/crawlers/coomerfans.pynew file mode 100644index 000000000..d6901e25c--- /dev/null+++ b/cyberdrop_dl/crawlers/coomerfans.py@@ -0,0 +1,127 @@+from __future__ import annotations++import dataclasses+import itertools+import re+from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text++if TYPE_CHECKING:+ from collections.abc import Generator++ from bs4 import BeautifulSoup++ from cyberdrop_dl.url_objects import ScrapeItem++_find_http_urls = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))").finditer++[email protected](slots=True)+class Post:+ date: str+ model_name: str+ content: str+ title: str+ images: tuple[str, ...]+ videos: tuple[str, ...]+++class CoomerFansCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Post": "/p/<post_id>/...",+ "User": "/u/<user_id>/...",+ "**NOTE**": "`--ignore-coomer-post-content` affects this crawler. All other kemono config options are ignored",+ }++ DOMAIN: ClassVar[str] = "coomerfans"+ FOLDER_DOMAIN: ClassVar[str] = "CoomerFans"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://coomerfans.com")+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {title}"+ NEXT_PAGE_SELECTOR: ClassVar[str] = ".pagination a.next"++ @classmethod+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ url = super().transform_url(url)+ match url.parts[1:]:+ case ["p", _, *rest] | ["u", _, _, *rest] if not rest:+ return url / ""+ case _:+ return url++ @property+ def ignore_content(self) -> bool:+ return self.manager.config.settings.ignore_options.ignore_coomer_post_content++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["p", post_id, *_]:+ return await self.post(scrape_item, post_id)+ case ["u", _service, _user_id, *_]:+ return await self.profile(scrape_item)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def post(self, scrape_item: ScrapeItem, post_id: str) -> None:+ soup = await self.request_soup(scrape_item.url)+ post = _parse_post(soup)+ scrape_item.setup_as_album(self.create_title(post.model_name))+ scrape_item.uploaded_at = date = self.parse_iso_date(post.date)+ post_title = self.create_separate_post_title(post.title, post_id, date)+ scrape_item.add_to_parent_title(post_title)+ self.create_task(self.write_metadata(scrape_item, f"post_{post_id}", post))+ self._post(scrape_item, post)++ def _post(self, scrape_item: ScrapeItem, post: Post) -> None:+ seen: set[str] = set()+ for url in itertools.chain(post.images, post.videos):+ if url not in seen:+ seen.add(url)+ self.create_task(self.direct_file(scrape_item, self.parse_url(url)))+ scrape_item.add_children()++ if not post.content or self.ignore_content:+ return++ for url in self.__parse_content_urls(post):+ self.handle_external_links(scrape_item.create_child(url))+ scrape_item.add_children()++ def __parse_content_urls(self, post: Post) -> Generator[AbsoluteHttpURL]:+ seen: set[str] = set()+ for match in _find_http_urls(post.content):+ if (link := match.group().replace(".md.", ".")) not in seen:+ seen.add(link)+ try:+ url = self.parse_url(link)+ except Exception:+ pass+ else:+ if self.DOMAIN not in url.host:+ yield url++ @error_handling_wrapper+ async def profile(self, scrape_item: ScrapeItem) -> None:+ async for soup in self.web_pager(scrape_item.url):+ for _, child in self.iter_children(scrape_item, soup, "a.view-post[href]"):+ self.create_task(self.run(child))+++def _parse_post(soup: BeautifulSoup) -> Post:+ main = css.select(soup, "main.content")+ body = css.select(main, ".post-body")++ def get(attr: str) -> str:+ return css.select_text(main, attr)++ return Post(+ title=get("h1"),+ model_name=get(".model-name"),+ content=get(".post-date + p"),+ date=extr_text(get(".post-date"), "Added ", " +"),+ images=tuple(css.iselect(body, "img", "src")),+ videos=tuple(css.iselect(body, "source", "src")),+ )diff --git a/tests/crawlers/test_cases/coomerfans.py b/tests/crawlers/test_cases/coomerfans.pynew file mode 100644index 000000000..18947c090--- /dev/null+++ b/tests/crawlers/test_cases/coomerfans.py@@ -0,0 +1,171 @@+DOMAIN = "coomerfans"+TEST_CASES = [+ (+ "https://coomerfans.com/p/63797958/315883/onlyfans",+ [+ {+ "url": "https://img1.coomerfans.com/storage/3/xt/qi/2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",+ "filename": "2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",+ "debrid_link": None,+ "original_filename": "2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",+ "referer": "https://coomerfans.com/p/63797958/315883/onlyfans",+ "uploaded_at": 1738836595,+ "download_folder": "re:carolynnmoon (CoomerFans)",+ }+ ],+ ),+ (+ "https://coomerfans.com/p/63797958",+ [+ {+ "url": "https://img1.coomerfans.com/storage/3/xt/qi/2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",+ "filename": "2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",+ "debrid_link": None,+ "original_filename": "2892ac-01963009-8b1f-725c-b601-d57777b4bb98.mp4",+ "referer": "re:https://coomerfans.com/p/63797958",+ "uploaded_at": 1738836595,+ "download_folder": "re:carolynnmoon (CoomerFans)",+ }+ ],+ ),+ (+ "https://coomerfans.com/p/73779832/338170/onlyfans",+ [+ {+ "url": "https://img1.coomerfans.com/storage/7/zn/iy/2892ac-01961036-199f-7926-873a-68444ae64ee8.jpg",+ "filename": "2892ac-01961036-199f-7926-873a-68444ae64ee8.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-199f-7926-873a-68444ae64ee8.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/10/sk/di/2892ac-01961036-19a0-72bc-997c-faa5b57afe74.jpg",+ "filename": "2892ac-01961036-19a0-72bc-997c-faa5b57afe74.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19a0-72bc-997c-faa5b57afe74.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/4/xb/qs/2892ac-01961036-19a0-730c-8c58-5a8378349587.jpg",+ "filename": "2892ac-01961036-19a0-730c-8c58-5a8378349587.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19a0-730c-8c58-5a8378349587.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/9/it/xf/2892ac-01961036-19a0-7424-ab8c-be28c49d235e.jpg",+ "filename": "2892ac-01961036-19a0-7424-ab8c-be28c49d235e.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19a0-7424-ab8c-be28c49d235e.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/2/yz/yc/2892ac-01961036-19a0-75b3-bed0-17b813e28931.jpg",+ "filename": "2892ac-01961036-19a0-75b3-bed0-17b813e28931.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19a0-75b3-bed0-17b813e28931.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/12/hm/hd/2892ac-01961036-19a0-7bd9-84a3-75a651ebfc3d.jpg",+ "filename": "2892ac-01961036-19a0-7bd9-84a3-75a651ebfc3d.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19a0-7bd9-84a3-75a651ebfc3d.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/1/lo/vd/2892ac-01961036-19a0-7bd5-9180-146ca7ac6b50.jpg",+ "filename": "2892ac-01961036-19a0-7bd5-9180-146ca7ac6b50.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19a0-7bd5-9180-146ca7ac6b50.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/2/sg/wb/2892ac-01961036-19a3-7d4e-854c-07a57e9c2975.jpg",+ "filename": "2892ac-01961036-19a3-7d4e-854c-07a57e9c2975.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19a3-7d4e-854c-07a57e9c2975.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/7/yh/zv/2892ac-01961036-19ab-7b46-bd88-ce9541d9e8f5.jpg",+ "filename": "2892ac-01961036-19ab-7b46-bd88-ce9541d9e8f5.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19ab-7b46-bd88-ce9541d9e8f5.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/7/sg/ot/2892ac-01961036-19bb-76ff-8a22-5c2e6aef03b2.jpg",+ "filename": "2892ac-01961036-19bb-76ff-8a22-5c2e6aef03b2.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19bb-76ff-8a22-5c2e6aef03b2.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/4/ns/ds/2892ac-01961036-19bd-741e-8ead-503329369493.jpg",+ "filename": "2892ac-01961036-19bd-741e-8ead-503329369493.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19bd-741e-8ead-503329369493.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/4/cf/mb/2892ac-01961036-19bd-7897-99d2-95d1d004874b.jpg",+ "filename": "2892ac-01961036-19bd-7897-99d2-95d1d004874b.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19bd-7897-99d2-95d1d004874b.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/7/on/zf/2892ac-01961036-19d3-7e4b-a226-b57dcc1e7581.jpg",+ "filename": "2892ac-01961036-19d3-7e4b-a226-b57dcc1e7581.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-19d3-7e4b-a226-b57dcc1e7581.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/6/gq/yr/2892ac-01961036-1a3b-73fb-9c95-125499caa2d6.jpg",+ "filename": "2892ac-01961036-1a3b-73fb-9c95-125499caa2d6.jpg",+ "debrid_link": None,+ "original_filename": "2892ac-01961036-1a3b-73fb-9c95-125499caa2d6.jpg",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ {+ "url": "https://img1.coomerfans.com/storage/7/fr/ob/2892ac-01970401-3ad7-73d5-bbec-6426c3edb74b.mp4",+ "filename": "2892ac-01970401-3ad7-73d5-bbec-6426c3edb74b.mp4",+ "debrid_link": None,+ "original_filename": "2892ac-01970401-3ad7-73d5-bbec-6426c3edb74b.mp4",+ "referer": "https://coomerfans.com/p/73779832/338170/onlyfans",+ "uploaded_at": 1717563216,+ "download_folder": "re:belledelphine (CoomerFans)",+ },+ ],+ ),+]