Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: adding new crawler (nudeleted.com) (#1974)
Barbarella66666669c2cb04bf0ef
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex 155cab848..629d82a73 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -59,6 +59,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):"Members": "/members/<member_id>",}NEXT_PAGE_SELECTOR: ClassVar[str] = "li.pagination-next > a"+ THUMBNAIL_SELECTOR: ClassVar[str] = Selector.THUMBNAILS_RATE_LIMIT: ClassVar[RateLimit] = 6, 5def __init_subclass__(cls, *, ensure_trailing_slash: bool = False, **kwargs: Any) -> None:@@ -150,9 +151,17 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):async def _iter_videos(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:async for soup in self.web_pager(url or scrape_item.url):- for new_scrape_item in self.iter_children(scrape_item, soup, Selector.THUMBNAILS):+ for new_scrape_item in self.iter_children(scrape_item, soup, self.THUMBNAIL_SELECTOR):self.create_task(self.run(new_scrape_item))+ def _extract_upload_date(self, soup: BeautifulSoup) -> int | None:+ try:+ date_str = css.json_ld(soup)["uploadDate"]+ except (LookupError, ValueError, css.SelectorError):+ return None++ return self.parse_iso_date(date_str)+@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item.url):@@ -161,15 +170,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):soup = await self.request_soup(scrape_item.url)video = extract_kvs_video(self, soup)filename, ext = self.get_filename_and_ext(video.url.name)-- try:- date_str = css.json_ld(soup)["uploadDate"]- except (LookupError, ValueError, css.SelectorError):- # Human date parsing was removed from parse_date. This fallback- # no longer supports relative strings like "2 hours ago".- pass- else:- scrape_item.uploaded_at = self.parse_iso_date(date_str)+ scrape_item.uploaded_at = self._extract_upload_date(soup)await self.handle_file(scrape_item.url,diff --git a/cyberdrop_dl/crawlers/nudeleted.py b/cyberdrop_dl/crawlers/nudeleted.pynew file mode 100644index 000000000..2da4d7245--- /dev/null+++ b/cyberdrop_dl/crawlers/nudeleted.py@@ -0,0 +1,40 @@+from __future__ import annotations++from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler+from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem+from cyberdrop_dl.utils import css++if TYPE_CHECKING:+ from bs4 import BeautifulSoup++ from cyberdrop_dl.crawlers.crawler import SupportedPaths+++class NudeletedCrawler(KernelVideoSharingCrawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Video": "/videos/...",+ "Tags": "/tags/...",+ "Search": "/search/...",+ }+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://nudeleted.com")+ DOMAIN: ClassVar[str] = "nudeleted"+ FOLDER_DOMAIN: ClassVar[str] = "Nudeleted"+ NEXT_PAGE_SELECTOR: ClassVar[str] = "li.next > a"+ THUMBNAIL_SELECTOR: ClassVar[str] = "div.margin-fix > div.item a"++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["videos", *_]:+ return await self.video(scrape_item)+ case ["search", query, *_]:+ return await self.search(scrape_item, query)+ case ["tags" as type_, name, *_]:+ return await self.collection(scrape_item, name, type_)+ case _:+ raise ValueError++ def _extract_upload_date(self, soup: BeautifulSoup) -> int | None:+ date_str: str = css.select(soup, 'meta[itemprop="uploadDate"]', "content")+ return self.parse_iso_date(date_str)diff --git a/tests/crawlers/test_cases/nudeleted.py b/tests/crawlers/test_cases/nudeleted.pynew file mode 100644index 000000000..7688b982b--- /dev/null+++ b/tests/crawlers/test_cases/nudeleted.py@@ -0,0 +1,41 @@+DOMAIN = "nudeleted"+TEST_CASES = [+ {+ "url": "https://nudeleted.com/videos/9920/normal-nude-and-dreads/",+ "results": [+ {+ "url": "https://nudeleted.com/videos/9920/normal-nude-and-dreads",+ "filename": "Normal nude. And dreads. [9920].mp4",+ "referer": "https://nudeleted.com/videos/9920/normal-nude-and-dreads/",+ "download_folder": "re:Loose Files (Nudeleted)",+ "uploaded_at": 1653579022,+ }+ ],+ },+ {+ "url": "https://nudeleted.com/tags/mayer-lil/",+ "results": [+ {+ "url": "https://nudeleted.com/videos/10434/mayer-lil-nip-slips-troughout",+ "referer": "https://nudeleted.com/videos/10434/mayer-lil-nip-slips-troughout",+ "album_id": None,+ "uploaded_at": int,+ "download_folder": r"re:mayer\-lil \[tag\] \(Nudeleted\)",+ },+ ],+ "count": range(20, 50),+ },+ {+ "url": "https://nudeleted.com/search/hitomi/",+ "results": [+ {+ "url": "https://nudeleted.com/videos/10960/hitomi-r18",+ "referer": "https://nudeleted.com/videos/10960/hitomi-r18",+ "album_id": None,+ "uploaded_at": int,+ "download_folder": r"re:hitomi \[search\] \(Nudeleted\)",+ },+ ],+ "count": range(8, 15),+ },+]