Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add clyp.it support (#1906)

NTFSvolumee9500b064a8f
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5aa1fdc27..ef6443b90 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Acast support
- Whyp.it support
- Soundgasm support
+- Clyp.it support
### Changed
diff --git a/cyberdrop_dl/crawlers/clyp_it.py b/cyberdrop_dl/crawlers/clyp_it.py
new file mode 100644
index 000000000..54371c8b6
--- /dev/null
+++ b/cyberdrop_dl/crawlers/clyp_it.py
@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import dataclasses
+import datetime
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from cyberdrop_dl import env
+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, auto_task_id
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import error_handling_wrapper, extr_text, parse_url
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator
+
+ from cyberdrop_dl.url_objects import ScrapeItem
+
+_PREMIUM_SUB_RELEASE_DATE = datetime.datetime(2017, 1, 1, tzinfo=datetime.UTC).timestamp()
+# Approx date from https://web.archive.org/web/20170520211342/https://clyp.it/premium-pricing
+
+
+class ClypItCrawler(Crawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Audio": "/<audio_id>",
+ "User": "/user/<user_id>",
+ }
+ DOMAIN: ClassVar[str] = "clyp.it"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://clyp.it")
+
+ def __post_init__(self) -> None:
+ self.api: ClypItAPI = ClypItAPI(self)
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case [audio_id]:
+ return await self.audio(scrape_item, audio_id)
+ case ["user", user_id]:
+ return await self.user(scrape_item, user_id)
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def user(self, scrape_item: ScrapeItem, user_id: str) -> None:
+ scrape_item.setup_as_profile("")
+ async for audios in self.api.user_uploads(user_id):
+ for audio in audios:
+ new_item = scrape_item.create_child(self.PRIMARY_URL / audio.id)
+ self.create_task(self._audio_task(new_item, audio))
+ scrape_item.add_children()
+
+ @error_handling_wrapper
+ async def audio(self, scrape_item: ScrapeItem, audio_id: str) -> None:
+ if await self.check_complete_from_referer(scrape_item.url):
+ return
+
+ audio = await self.api.audio(audio_id, token=scrape_item.url.query.get("token"))
+ await self._audio(scrape_item, audio)
+
+ async def _audio(self, scrape_item: ScrapeItem, audio: Audio) -> None:
+ scrape_item.setup_as_profile(self.create_title(audio.user.full_name, audio.user.id))
+ scrape_item.uploaded_at = date = self.parse_iso_date(audio.created_at)
+ src = audio.mp3
+ if date > _PREMIUM_SUB_RELEASE_DATE:
+ src = await self.api.wav(audio.id, token=scrape_item.url.query.get("token")) or src
+
+ _, ext = self.get_filename_and_ext(src.name)
+ filename = self.create_custom_filename(audio.title, ext, file_id=audio.id)
+ await self.handle_file(src, scrape_item, audio.title, ext, custom_filename=filename, metadata=audio)
+
+ @auto_task_id
+ @error_handling_wrapper
+ async def _audio_task(self, scrape_item: ScrapeItem, audio: Audio) -> None:
+ if await self.check_complete_from_referer(scrape_item.url):
+ return
+
+ await self._audio(scrape_item, audio)
+
+
+class ClypItAPI(API):
+ # docs: https://clyp.it/api
+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api.clyp.it")
+
+ async def audio(self, audio_id: str, token: str | None) -> Audio:
+ api_url = self.ENTRYPOINT / audio_id / "Playlist"
+ if token:
+ api_url = api_url.with_query(token=token)
+
+ playlist = await self.request_json(api_url)
+ audios = map(_parse_audio, playlist["AudioFiles"])
+ return next(a for a in audios if a.id == audio_id)
+
+ async def wav(self, audio_id: str, token: str | None) -> AbsoluteHttpURL | None:
+ if env.CLYPIT_PREFER_MP3:
+ return None
+ url = self.crawler.PRIMARY_URL / audio_id
+ if token:
+ url = url.with_query(token=token)
+ text = await self.request_text(url)
+ try:
+ src = extr_text(text, 'var wavStreamUrl = "', '";')
+ except ValueError:
+ return None
+ else:
+ return self.crawler.parse_url(src)
+
+ def user_uploads(self, user_id: str) -> AsyncGenerator[map[Audio]]:
+ api_url = self.ENTRYPOINT / "User" / user_id / "Uploads"
+ return self._pager(api_url)
+
+ async def _pager(self, url: AbsoluteHttpURL) -> AsyncGenerator[map[Audio]]:
+ url = url.update_query(count=20)
+ while True:
+ resp = await self.request_json(url)
+ yield map(_parse_audio, resp["Data"])
+ next_page: str | None = resp.get("Paging").get("Next")
+ if not next_page:
+ break
+ url = self.crawler.parse_url(next_page)
+
+
[email protected](slots=True)
+class User:
+ id: str
+ first_name: str
+ last_name: str | None = None
+
+ @property
+ def full_name(self) -> str:
+ return " ".join(filter(None, (self.first_name, self.last_name)))
+
+
[email protected](slots=True)
+class Audio:
+ id: str
+ title: str
+ created_at: str
+ user: User
+ mp3: AbsoluteHttpURL
+
+
+def _parse_audio(audio: dict[str, Any]) -> Audio:
+ return Audio(
+ id=audio["AudioFileId"],
+ title=audio["Title"],
+ created_at=audio["DateCreated"],
+ mp3=parse_url(audio.get("SecureMp3Url") or audio["Mp3Url"]),
+ user=_parse_user(audio["User"]),
+ )
+
+
+def _parse_user(user: dict[str, Any]) -> User:
+ return User(id=user["UserId"], first_name=user["FirstName"], last_name=user.get("LastName"))
diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.py
index b37b15304..80ad4a34c 100644
--- a/cyberdrop_dl/env.py
+++ b/cyberdrop_dl/env.py
@@ -43,6 +43,7 @@ PIXELDRAIN_PROXY = _env("PIXELDRAIN_PROXY")
BANDCAMP_FORMATS = _env("BANDCAMP_FORMATS")
ENABLE_TWITTER = bool(_env("ENABLE_TWITTER"))
ONEPACE_PREFER_DUB = bool(_env("ONEPACE_PREFER_DUB"))
+CLYPIT_PREFER_MP3 = bool(_env("CLYPIT_PREFER_MP3"))
ALL_VARS = dict(sorted(ALL_VARS.items())) # pyright: ignore[reportConstantRedefinition]
ALL_VARS_RESOLVED = dict(
diff --git a/tests/crawlers/test_cases/clyp_it.py b/tests/crawlers/test_cases/clyp_it.py
new file mode 100644
index 000000000..9b78111f2
--- /dev/null
+++ b/tests/crawlers/test_cases/clyp_it.py
@@ -0,0 +1,47 @@
+DOMAIN = "clyp.it"
+TEST_CASES = [
+ {
+ "url": "https://clyp.it/x1ujleej",
+ "results": [
+ {
+ "url": "re:https://audio.clyp.it/x1ujleej.wav?",
+ "filename": "Monkeys favourite songs MASHUP! [x1ujleej].wav",
+ "debrid_link": None,
+ "original_filename": "Monkey's favourite songs MASHUP!",
+ "referer": "https://clyp.it/x1ujleej",
+ "album_id": None,
+ "uploaded_at": 1684226059,
+ "download_folder": "re:Valentin Brunn (Clyp.it)",
+ }
+ ],
+ "count": 1,
+ },
+ {
+ "url": "https://clyp.it/b04p1odi?token=b0078e077e15835845c528a44417719d",
+ "results": [
+ {
+ "url": "re:https://audio.clyp.it/b04p1odi.mp3",
+ "filename": "GJ! (Reward Edit) [b04p1odi].mp3",
+ "debrid_link": None,
+ "original_filename": "GJ! (Reward Edit)",
+ "referer": "https://clyp.it/b04p1odi?token=b0078e077e15835845c528a44417719d",
+ "album_id": None,
+ "uploaded_at": 1528241278,
+ "download_folder": "re:BABYMETAL Songs (Clyp.it)",
+ }
+ ],
+ "count": 1,
+ },
+ {
+ "url": "https://clyp.it/user/gxslmwpj",
+ "results": [
+ {
+ "url": "ANY",
+ "album_id": None,
+ "uploaded_at": int,
+ "download_folder": "re:Valentin Brunn (Clyp.it)",
+ },
+ ],
+ "count": 239,
+ },
+]