Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: adding new crawler (nudeleted.com) (#1974)

Barbarella66666669c2cb04bf0ef
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.py
index 155cab848..629d82a73 100644
--- a/cyberdrop_dl/crawlers/_kvs.py
+++ b/cyberdrop_dl/crawlers/_kvs.py
@@ -59,6 +59,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
"Members": "/members/<member_id>",
}
NEXT_PAGE_SELECTOR: ClassVar[str] = "li.pagination-next > a"
+ THUMBNAIL_SELECTOR: ClassVar[str] = Selector.THUMBNAILS
_RATE_LIMIT: ClassVar[RateLimit] = 6, 5
def __init_subclass__(cls, *, ensure_trailing_slash: bool = False, **kwargs: Any) -> None:
@@ -150,9 +151,17 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
async def _iter_videos(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:
async for soup in self.web_pager(url or scrape_item.url):
- for new_scrape_item in self.iter_children(scrape_item, soup, Selector.THUMBNAILS):
+ for new_scrape_item in self.iter_children(scrape_item, soup, self.THUMBNAIL_SELECTOR):
self.create_task(self.run(new_scrape_item))
+ def _extract_upload_date(self, soup: BeautifulSoup) -> int | None:
+ try:
+ date_str = css.json_ld(soup)["uploadDate"]
+ except (LookupError, ValueError, css.SelectorError):
+ return None
+
+ return self.parse_iso_date(date_str)
+
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item.url):
@@ -161,15 +170,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
soup = await self.request_soup(scrape_item.url)
video = extract_kvs_video(self, soup)
filename, ext = self.get_filename_and_ext(video.url.name)
-
- try:
- date_str = css.json_ld(soup)["uploadDate"]
- except (LookupError, ValueError, css.SelectorError):
- # Human date parsing was removed from parse_date. This fallback
- # no longer supports relative strings like "2 hours ago".
- pass
- else:
- scrape_item.uploaded_at = self.parse_iso_date(date_str)
+ scrape_item.uploaded_at = self._extract_upload_date(soup)
await self.handle_file(
scrape_item.url,
diff --git a/cyberdrop_dl/crawlers/nudeleted.py b/cyberdrop_dl/crawlers/nudeleted.py
new file mode 100644
index 000000000..2da4d7245
--- /dev/null
+++ b/cyberdrop_dl/crawlers/nudeleted.py
@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler
+from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
+from cyberdrop_dl.utils import css
+
+if TYPE_CHECKING:
+ from bs4 import BeautifulSoup
+
+ from cyberdrop_dl.crawlers.crawler import SupportedPaths
+
+
+class NudeletedCrawler(KernelVideoSharingCrawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Video": "/videos/...",
+ "Tags": "/tags/...",
+ "Search": "/search/...",
+ }
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://nudeleted.com")
+ DOMAIN: ClassVar[str] = "nudeleted"
+ FOLDER_DOMAIN: ClassVar[str] = "Nudeleted"
+ NEXT_PAGE_SELECTOR: ClassVar[str] = "li.next > a"
+ THUMBNAIL_SELECTOR: ClassVar[str] = "div.margin-fix > div.item a"
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["videos", *_]:
+ return await self.video(scrape_item)
+ case ["search", query, *_]:
+ return await self.search(scrape_item, query)
+ case ["tags" as type_, name, *_]:
+ return await self.collection(scrape_item, name, type_)
+ case _:
+ raise ValueError
+
+ def _extract_upload_date(self, soup: BeautifulSoup) -> int | None:
+ date_str: str = css.select(soup, 'meta[itemprop="uploadDate"]', "content")
+ return self.parse_iso_date(date_str)
diff --git a/tests/crawlers/test_cases/nudeleted.py b/tests/crawlers/test_cases/nudeleted.py
new file mode 100644
index 000000000..7688b982b
--- /dev/null
+++ b/tests/crawlers/test_cases/nudeleted.py
@@ -0,0 +1,41 @@
+DOMAIN = "nudeleted"
+TEST_CASES = [
+ {
+ "url": "https://nudeleted.com/videos/9920/normal-nude-and-dreads/",
+ "results": [
+ {
+ "url": "https://nudeleted.com/videos/9920/normal-nude-and-dreads",
+ "filename": "Normal nude. And dreads. [9920].mp4",
+ "referer": "https://nudeleted.com/videos/9920/normal-nude-and-dreads/",
+ "download_folder": "re:Loose Files (Nudeleted)",
+ "uploaded_at": 1653579022,
+ }
+ ],
+ },
+ {
+ "url": "https://nudeleted.com/tags/mayer-lil/",
+ "results": [
+ {
+ "url": "https://nudeleted.com/videos/10434/mayer-lil-nip-slips-troughout",
+ "referer": "https://nudeleted.com/videos/10434/mayer-lil-nip-slips-troughout",
+ "album_id": None,
+ "uploaded_at": int,
+ "download_folder": r"re:mayer\-lil \[tag\] \(Nudeleted\)",
+ },
+ ],
+ "count": range(20, 50),
+ },
+ {
+ "url": "https://nudeleted.com/search/hitomi/",
+ "results": [
+ {
+ "url": "https://nudeleted.com/videos/10960/hitomi-r18",
+ "referer": "https://nudeleted.com/videos/10960/hitomi-r18",
+ "album_id": None,
+ "uploaded_at": int,
+ "download_folder": r"re:hitomi \[search\] \(Nudeleted\)",
+ },
+ ],
+ "count": range(8, 15),
+ },
+]