Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: more crawler fixes
NTFSvolume067556aa2ce2
diff --git a/cyberdrop_dl/crawlers/realbooru.py b/cyberdrop_dl/crawlers/realbooru.pyindex 53188fac2..21098aa99 100644--- a/cyberdrop_dl/crawlers/realbooru.py+++ b/cyberdrop_dl/crawlers/realbooru.py@@ -1,7 +1,8 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, ClassVar, final+from cyberdrop_dl.clients.http import HTTPConfigfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css@@ -11,11 +12,13 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem+@finalclass Selector:CONTENT = "div[class=items] div a"MEDIA = "img#image, video source"+@HTTPConfig(rate_limit=(3, 1))class RealBooruCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"File": "?id=<file_id>",diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.pyindex bef4fee75..23e176acf 100644--- a/cyberdrop_dl/crawlers/rumble.py+++ b/cyberdrop_dl/crawlers/rumble.py@@ -6,6 +6,7 @@ from enum import IntEnumfrom typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl import aio+from cyberdrop_dl.clients.http import HTTPConfigfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import DownloadError, ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolution, Subtitle@@ -61,6 +62,7 @@ class Video:subtitles: tuple[Subtitle, ...]+@HTTPConfig(impersonate="firefox", rate_limit=(8, 1))class RumbleCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Channel": "/c/<name>",diff --git a/tests/crawlers/test_cases/rule34vault.py b/tests/crawlers/test_cases/rule34vault.pyindex a3a763366..7e15f5cf2 100644--- a/tests/crawlers/test_cases/rule34vault.py+++ b/tests/crawlers/test_cases/rule34vault.py@@ -25,7 +25,7 @@ TEST_CASES = ["download_folder": "re:하츠네 미쿠 [playlist] (Rule34Vault)",},],- "count": 33,+ "count": range(78, 90),},{"url": "https://rule34vault.com/sarahvividart%7Calien",diff --git a/tests/crawlers/test_cases/rule34video.py b/tests/crawlers/test_cases/rule34video.pyindex 25df876af..96066863b 100644--- a/tests/crawlers/test_cases/rule34video.py+++ b/tests/crawlers/test_cases/rule34video.py@@ -19,6 +19,6 @@ TEST_CASES = ["results": [{"url": "ANY", "album_id": None, "uploaded_at": int, "download_folder": "re:Lozi3d [members] (Rule34Video)"}],- "count": 11,+ "count": range(23, 30),},]diff --git a/tests/crawlers/test_cases/rumble.py b/tests/crawlers/test_cases/rumble.pyindex 87eced2b5..142d8c662 100644--- a/tests/crawlers/test_cases/rumble.py+++ b/tests/crawlers/test_cases/rumble.py@@ -1,26 +1,5 @@DOMAIN = "rumble"TEST_CASES = [- {- "url": "https://rumble.com/v71komc-home-alone-1990.html",- "results": [- {- "url": "https://rumble.com/hls-vod/6ze0m4/playlist.m3u8",- "filename": "Home Alone (1990) [v6ze0m4][1384p].mp4",- "original_filename": "Home Alone (1990).mp4",- "referer": "https://rumble.com/v71komc-home-alone-1990.html",- "album_id": None,- "uploaded_at": 1762902545,- },- {- "url": "https://1a-1791.com/video/fww1/4d/s8/11/K/A/T/y/KATyz.fn8Si.vtt",- "filename": "Home Alone (1990) [v6ze0m4][1384p].en.auto.vtt",- "original_filename": "English (auto)",- "referer": "https://rumble.com/v71komc-home-alone-1990.html#Home Alone (1990) [v6ze0m4][1384p].en.auto.vtt",- "album_id": None,- "uploaded_at": 1762902545,- },- ],- },{"url": "https://rumble.com/vwu8gp-residente-bzrp-music-sessions-49.html?e9s=src_v1_s","results": [@@ -38,18 +17,19 @@ TEST_CASES = ["url": "https://rumble.com/c/RumbleEvents","results": [{- "url": "https://1a-1791.com/video/s8/2/5/s/Q/8/5sQ8e.aaa.rec.mp4",+ "url": "https://1a-1791.com/video/vo/47ww/m03gQsmi4GE/Px13ulRwHFM.mp4","filename": "CLOSING OUT Dave Rubin & Benny Johnson LIVE - Rumble Exclusive Live at TPU [v1at1kt][1080p].mp4","referer": "https://rumble.com/v1df7i1-rumble-exclusive-live-at-tpusa.html","album_id": None,+ "download_folder": "re:RumbleEvents (Rumble)",},{- "url": "https://1a-1791.com/video/s8/2/p/W/J/7/pWJ7e.aaa.rec.mp4",+ "url": "https://1a-1791.com/video/vo/y7pe/dPdI3HbQ11Q/kkxZS05zoL0.mp4","filename": "Turning Point USA 2022 Student Action Summit DAY 1 - Rumble Exclusive Liv [v1amumr][1080p].mp4","referer": "https://rumble.com/v1d90jz-rumble-exclusive-live-at-tpusa.html",},{- "url": "https://1a-1791.com/video/s8/2/b/f/f/8/bff8e.aaa.rec.mp4",+ "url": "https://1a-1791.com/video/vo/5j7n/fsuRaNtWx5A/K9ZKZmzRXAg.mp4","filename": "President Donald J. Trump LIVE at Turning Point - Rumble Exclusive Live at [v1apryd][1080p].mp4","referer": "https://rumble.com/v1dbxvl-rumble-exclusive-live-at-tpusa.html",},diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.pyindex d2adb5fc2..8a932d7e9 100644--- a/tests/crawlers/test_crawlers.py+++ b/tests/crawlers/test_crawlers.py@@ -97,9 +97,9 @@ NOT_NONE = _NOT_NONE()def _validate_results(crawler: Crawler, test_case: test_cases.CrawlerTestCase, results: list[MediaItem]) -> None:- expected_results = sorted(test_case.results, key=lambda x: x["url"])+ expected_results = dict(sorted(((x["url"], idx), x) for idx, x in enumerate(test_case.results, 1)))origin = getattr(crawler, "PRIMARY_URL", AbsoluteHttpURL("https://google.com"))- for index, (expected, media_item) in enumerate(zip(expected_results, results, strict=False), 1):+ for (index, expected), media_item in zip(expected_results.items(), results, strict=False):for attr_name, expected_value in expected.items():result_value = getattr(media_item, attr_name)if isinstance(result_value, Path):