Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add livestreamfails support (#1909)
NTFSvolumeadc0e79cf191
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ef6443b90..d02166b84 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Whyp.it support- Soundgasm support- Clyp.it support+- Livestreamfails support### Changeddiff --git a/cyberdrop_dl/crawlers/livestreamfails.py b/cyberdrop_dl/crawlers/livestreamfails.pynew file mode 100644index 000000000..9b8afbc45--- /dev/null+++ b/cyberdrop_dl/crawlers/livestreamfails.py@@ -0,0 +1,117 @@+from __future__ import annotations++import dataclasses+from typing import TYPE_CHECKING, Any, ClassVar++from cyberdrop_dl.crawlers.crawler import API, Crawler, RateLimit, SupportedPaths+from cyberdrop_dl.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import deserialize, error_handling_wrapper++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator++ from cyberdrop_dl.url_objects import ScrapeItem+++class _CDN:+ IMAGE = AbsoluteHttpURL("https://media-prod.livestreamfails.com/image")+ VIDEO = AbsoluteHttpURL("https://livestreamfails-video-prod.b-cdn.net/video")+++class LivestreamFailsCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Clip": "/clip/<video_id>",+ "Streamer": "/streamer/<streamer_id>",+ }+ DOMAIN: ClassVar[str] = "livestreamfails.com"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://livestreamfails.com")+ _RATE_LIMIT: ClassVar[RateLimit] = 8, 1++ def __post_init__(self) -> None:+ self.api: LivestreamFailsAPI = LivestreamFailsAPI(self)++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["clip", clip_id]:+ return await self.clip(scrape_item, clip_id)+ case ["streamer", streamer_id]:+ return await self.streamer(scrape_item, streamer_id)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def streamer(self, scrape_item: ScrapeItem, streamer_id: str) -> None:+ scrape_item.setup_as_profile("")+ async for videos in self.api.streamer_clips(streamer_id):+ for video in videos:+ url = self.PRIMARY_URL / f"clip/{video.id}"+ new_item = scrape_item.create_child(url)+ self.create_task(self._clip(new_item, video))+ scrape_item.add_children()++ @error_handling_wrapper+ async def clip(self, scrape_item: ScrapeItem, clip_id: str) -> None:+ if await self.check_complete_from_referer(scrape_item.url):+ return++ video = await self.api.clip(clip_id)+ await self._clip(scrape_item, video)++ @error_handling_wrapper+ async def _clip(self, scrape_item: ScrapeItem, clip: Clip) -> None:+ scrape_item.setup_as_album(self.create_title(clip.streamer.label, str(clip.streamer.id)))+ scrape_item.uploaded_at = self.parse_iso_date(clip.createdAt)+ _, ext = self.get_filename_and_ext(clip.src.name)+ filename = self.create_custom_filename(clip.label, ext, file_id=str(clip.id))+ await self.handle_file(clip.src, scrape_item, clip.label, ext, custom_filename=filename, metadata=clip)++[email protected](slots=True)+class Clip:+ id: int+ label: str+ createdAt: str # noqa: N815+ streamer: Streamer+ src: AbsoluteHttpURL+ thumbnail: AbsoluteHttpURL++[email protected](slots=True)+class Streamer:+ id: int+ label: str+++class LivestreamFailsAPI(API):+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api.livestreamfails.com")++ async def clip(self, clip_id: str) -> Clip:+ api_url = self.ENTRYPOINT / "clip" / clip_id+ video: dict[str, Any] = await self.request_json(api_url)+ return _parse_video(video)++ def streamer_clips(self, streamer_id: str) -> AsyncGenerator[map[Clip]]:+ api_url = self.ENTRYPOINT / "streamer" / streamer_id / "clips"+ return self._pager(api_url)++ async def _pager(self, url: AbsoluteHttpURL) -> AsyncGenerator[map[Clip]]:+ url = url.update_query(querySort="new")+ while True:+ resp = await self.request_json(url)+ if not resp:+ break+ last_id: str = resp[-1]["id"]+ yield map(_parse_video, resp)+ if len(resp) < 20:+ break+ url = url.update_query(queryAfter=last_id)+++def _parse_video(resp: dict[str, Any]) -> Clip:+ return deserialize(+ Clip,+ resp,+ src=_CDN.VIDEO / resp["videoId"],+ thumbnail=_CDN.IMAGE / resp["imageId"],+ streamer=deserialize(Streamer, resp["streamer"]),+ )diff --git a/tests/crawlers/test_cases/livestreamfails.py b/tests/crawlers/test_cases/livestreamfails.pynew file mode 100644index 000000000..96b70eddb--- /dev/null+++ b/tests/crawlers/test_cases/livestreamfails.py@@ -0,0 +1,45 @@+DOMAIN = "livestreamfails.com"+TEST_CASES = [+ {+ "url": "https://livestreamfails.com/clip/179468",+ "results": [+ {+ "url": "https://livestreamfails-video-prod.b-cdn.net/video/7c1c6131244daeaeaa2aee88e5dff03c54944901.mp4",+ "filename": "Ariathome runs into 2025 US Beatbox Champion (BizKit) [179468].mp4",+ "debrid_link": None,+ "original_filename": "Ariathome runs into 2025 US Beatbox Champion (BizKit)",+ "referer": "https://livestreamfails.com/clip/179468",+ "album_id": None,+ "uploaded_at": 1775626544,+ "download_folder": "re:ARIatHOME (Livestreamfails.com)",+ }+ ],+ "count": 1,+ },+ {+ "url": "https://livestreamfails.com/streamer/11261",+ "results": [+ {+ "url": "https://livestreamfails-video-prod.b-cdn.net/video/7b75a6775160fa346757cf6aca62aca2180ccfe3.mp4",+ "filename": "The most Insane Ice Bath [178966].mp4",+ "debrid_link": None,+ "original_filename": "The most Insane Ice Bath",+ "referer": "https://livestreamfails.com/clip/178966",+ "album_id": None,+ "uploaded_at": 1770664483,+ "download_folder": "re:KattyCore (Livestreamfails.com)",+ },+ {+ "url": "https://livestreamfails-video-prod.b-cdn.net/video/d0a0c77a026cf5df9624c70cf4ee6b7ab0e601c0.mp4",+ "filename": "Huge Snowman Reveal [178623].mp4",+ "debrid_link": None,+ "original_filename": "Huge Snowman Reveal",+ "referer": "https://livestreamfails.com/clip/178623",+ "album_id": None,+ "uploaded_at": 1767287443,+ "download_folder": "re:KattyCore (Livestreamfails.com)",+ },+ ],+ "count": 2,+ },+]