Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: always use permalink ID for filenames (Rumble)
NTFSvolume113404267b61
diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.pyindex 2bac084b6..bd95176dc 100644--- a/cyberdrop_dl/crawlers/rumble.py+++ b/cyberdrop_dl/crawlers/rumble.py@@ -9,7 +9,7 @@ from bs4 import BeautifulSoupfrom cyberdrop_dl import aiofrom cyberdrop_dl.clients.http import HTTPConfig-from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, auto_task_idfrom cyberdrop_dl.exceptions import DDOSGuardError, ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolution, Subtitlefrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -61,11 +61,15 @@ class Video:id: strtitle: strupload_date: str- url: AbsoluteHttpURL+ permalink: AbsoluteHttpURLformats: tuple[Format, ...]subtitles: tuple[Subtitle, ...]thumb: str | None = None+ @property+ def permalink_id(self) -> str:+ return self.permalink.name.partition("-")[0]+@HTTPConfig(impersonate="firefox", rate_limit=(8, 1))class RumbleCrawler(Crawler):@@ -137,8 +141,8 @@ class RumbleCrawler(Crawler):found_videos: bool = Falsefor video in map(_parse_video_obj, _find_video_objs(soup)):found_videos = True- new_item = scrape_item.create_child(video.url)- self.create_eager_task(self._video(new_item, video))+ new_item = scrape_item.create_child(video.permalink)+ self.create_eager_task(self.video_task(new_item, video))scrape_item.add_children()if found_videos:@@ -146,13 +150,9 @@ class RumbleCrawler(Crawler):# Fallback if we made the request with Flaresolverr. Video objects in the HTML are destroyed after JS loadsfor new_item in self.iter_children(scrape_item, soup, "rum-video-thumbnail a"):- found_videos = Trueself.create_task(self.run(new_item))scrape_item.add_children()- if not found_videos:- raise ScrapeError(422, "Did not find any videos on the last page")-@error_handling_wrapperasync def short(self, scrape_item: ScrapeItem, short_id: str) -> None:if await self.check_complete_from_referer(scrape_item.url):@@ -176,22 +176,33 @@ class RumbleCrawler(Crawler):@error_handling_wrapperasync def embed(self, scrape_item: ScrapeItem, embed_id: str) -> None:video = await self.api.embed(embed_id)+ if await self.check_complete_from_referer(video.permalink):+ returnawait self._video(scrape_item, video)@error_handling_wrapper- async def _video(self, scrape_item: ScrapeItem, video: Video) -> None:- if await self.check_complete_from_referer(scrape_item.url):+ @auto_task_id+ async def video_task(self, scrape_item: ScrapeItem, video: Video) -> None:+ if await self.check_complete_from_referer(video.permalink):return+ await self._video(scrape_item, video)++ async def _video(self, scrape_item: ScrapeItem, video: Video) -> None:best_format = max(await self._resolve_formats(video.formats))if best_format.m3u8:ext = ".mp4"else:_, ext = self.get_filename_and_ext(best_format.url.name)- video_name = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=best_format.resolution)+ video_name = self.create_custom_filename(+ video.title,+ ext,+ file_id=video.permalink_id,+ resolution=best_format.resolution,+ )scrape_item.uploaded_at = self.parse_iso_date(video.upload_date)- scrape_item.url = video.url+ scrape_item.url = video.permalinkself.create_eager_task(self.handle_file(best_format.url,@@ -233,7 +244,7 @@ class RumbleAPI(API):id=embed_id,upload_date=data["pubDate"],title=css.unescape(data["title"]),- url=self.parse_url(data["l"]),+ permalink=self.parse_url(data["l"]),formats=tuple(_parse_embed_formats(data.get("ua") or {})),subtitles=tuple(_parse_subs(data.get("cc") or {})),)@@ -246,14 +257,14 @@ class RumbleAPI(API):raise ScrapeError(422, "Unable to find short data")- async def get_embed_id(self, video_url: AbsoluteHttpURL) -> str:+ async def embed_id(self, video_url: AbsoluteHttpURL) -> str:oembed_url = (self.PRIMARY_URL / "api/Media/oembed.json").with_query(url=str(video_url))resp = await self.request_json(oembed_url)soup = BeautifulSoup(resp["html"], "html.parser")return self.parse_url(css.select(soup, "iframe", "src")).name- async def resolve(self, url: AbsoluteHttpURL) -> Video:- embed_id = await self.get_embed_id(url)+ async def resolve(self, permalink: AbsoluteHttpURL) -> Video:+ embed_id = await self.embed_id(permalink)return await self.embed(embed_id)@@ -262,7 +273,7 @@ def _parse_video_obj(short: dict[str, Any]) -> Video:id=short["permalink_id"],upload_date=short["upload_date"],title=css.unescape(short["title"]),- url=RumbleCrawler.parse_url(short["url"]),+ permalink=RumbleCrawler.parse_url(short["url"]),formats=tuple(_parse_video_obj_formats(short["videos"])),subtitles=(),thumb=short.get("thumb"),diff --git a/tests/crawlers/test_cases/rumble.py b/tests/crawlers/test_cases/rumble.pyindex 7ff660746..c49ab0ecc 100644--- a/tests/crawlers/test_cases/rumble.py+++ b/tests/crawlers/test_cases/rumble.py@@ -23,7 +23,7 @@ TEST_CASES = ["results": [{"url": "re:/video/s8/2/z/z/A/s/zzAsd.caa.mp4",- "filename": "RESIDENTE -- BZRP Music Sessions #49 [vu82er][360p].mp4",+ "filename": "RESIDENTE -- BZRP Music Sessions #49 [vwu8gp][360p].mp4","original_filename": "RESIDENTE -- BZRP Music Sessions #49.mp4","referer": "https://rumble.com/vwu8gp-residente-bzrp-music-sessions-49.html","album_id": None,@@ -36,19 +36,19 @@ TEST_CASES = ["results": [{"url": "re:/video/vo/47ww/m03gQsmi4GE/Px13ulRwHFM.mp4",- "filename": "CLOSING OUT Dave Rubin & Benny Johnson LIVE - Rumble Exclusive Live at TPU [v1at1kt][1080p].mp4",+ "filename": "CLOSING OUT Dave Rubin & Benny Johnson LIVE - Rumble Exclusive Live at TPU [v1df7i1][1080p].mp4","referer": "https://rumble.com/v1df7i1-rumble-exclusive-live-at-tpusa.html","album_id": None,"download_folder": "re:RumbleEvents (Rumble)",},{"url": "re:/video/vo/y7pe/dPdI3HbQ11Q/kkxZS05zoL0.mp4",- "filename": "Turning Point USA 2022 Student Action Summit DAY 1 - Rumble Exclusive Liv [v1amumr][1080p].mp4",+ "filename": "Turning Point USA 2022 Student Action Summit DAY 1 - Rumble Exclusive Live [v1d90jz][1080p].mp4","referer": "https://rumble.com/v1d90jz-rumble-exclusive-live-at-tpusa.html",},{"url": "re:/video/vo/5j7n/fsuRaNtWx5A/K9ZKZmzRXAg.mp4",- "filename": "President Donald J. Trump LIVE at Turning Point - Rumble Exclusive Live at [v1apryd][1080p].mp4",+ "filename": "President Donald J. Trump LIVE at Turning Point - Rumble Exclusive Live at [v1dbxvl][1080p].mp4","referer": "https://rumble.com/v1dbxvl-rumble-exclusive-live-at-tpusa.html",},],