Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: more crawler fixes

NTFSvolume067556aa2ce2
diff --git a/cyberdrop_dl/crawlers/realbooru.py b/cyberdrop_dl/crawlers/realbooru.py
index 53188fac2..21098aa99 100644
--- a/cyberdrop_dl/crawlers/realbooru.py
+++ b/cyberdrop_dl/crawlers/realbooru.py
@@ -1,7 +1,8 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, final
+from cyberdrop_dl.clients.http import HTTPConfig
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css
@@ -11,11 +12,13 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
+@final
class Selector:
CONTENT = "div[class=items] div a"
MEDIA = "img#image, video source"
+@HTTPConfig(rate_limit=(3, 1))
class RealBooruCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"File": "?id=<file_id>",
diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.py
index bef4fee75..23e176acf 100644
--- a/cyberdrop_dl/crawlers/rumble.py
+++ b/cyberdrop_dl/crawlers/rumble.py
@@ -6,6 +6,7 @@ from enum import IntEnum
from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl import aio
+from cyberdrop_dl.clients.http import HTTPConfig
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import DownloadError, ScrapeError
from cyberdrop_dl.mediaprops import Resolution, Subtitle
@@ -61,6 +62,7 @@ class Video:
subtitles: tuple[Subtitle, ...]
+@HTTPConfig(impersonate="firefox", rate_limit=(8, 1))
class RumbleCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Channel": "/c/<name>",
diff --git a/tests/crawlers/test_cases/rule34vault.py b/tests/crawlers/test_cases/rule34vault.py
index a3a763366..7e15f5cf2 100644
--- a/tests/crawlers/test_cases/rule34vault.py
+++ b/tests/crawlers/test_cases/rule34vault.py
@@ -25,7 +25,7 @@ TEST_CASES = [
"download_folder": "re:하츠네 미쿠 [playlist] (Rule34Vault)",
},
],
- "count": 33,
+ "count": range(78, 90),
},
{
"url": "https://rule34vault.com/sarahvividart%7Calien",
diff --git a/tests/crawlers/test_cases/rule34video.py b/tests/crawlers/test_cases/rule34video.py
index 25df876af..96066863b 100644
--- a/tests/crawlers/test_cases/rule34video.py
+++ b/tests/crawlers/test_cases/rule34video.py
@@ -19,6 +19,6 @@ TEST_CASES = [
"results": [
{"url": "ANY", "album_id": None, "uploaded_at": int, "download_folder": "re:Lozi3d [members] (Rule34Video)"}
],
- "count": 11,
+ "count": range(23, 30),
},
]
diff --git a/tests/crawlers/test_cases/rumble.py b/tests/crawlers/test_cases/rumble.py
index 87eced2b5..142d8c662 100644
--- a/tests/crawlers/test_cases/rumble.py
+++ b/tests/crawlers/test_cases/rumble.py
@@ -1,26 +1,5 @@
DOMAIN = "rumble"
TEST_CASES = [
- {
- "url": "https://rumble.com/v71komc-home-alone-1990.html",
- "results": [
- {
- "url": "https://rumble.com/hls-vod/6ze0m4/playlist.m3u8",
- "filename": "Home Alone (1990) [v6ze0m4][1384p].mp4",
- "original_filename": "Home Alone (1990).mp4",
- "referer": "https://rumble.com/v71komc-home-alone-1990.html",
- "album_id": None,
- "uploaded_at": 1762902545,
- },
- {
- "url": "https://1a-1791.com/video/fww1/4d/s8/11/K/A/T/y/KATyz.fn8Si.vtt",
- "filename": "Home Alone (1990) [v6ze0m4][1384p].en.auto.vtt",
- "original_filename": "English (auto)",
- "referer": "https://rumble.com/v71komc-home-alone-1990.html#Home Alone (1990) [v6ze0m4][1384p].en.auto.vtt",
- "album_id": None,
- "uploaded_at": 1762902545,
- },
- ],
- },
{
"url": "https://rumble.com/vwu8gp-residente-bzrp-music-sessions-49.html?e9s=src_v1_s",
"results": [
@@ -38,18 +17,19 @@ TEST_CASES = [
"url": "https://rumble.com/c/RumbleEvents",
"results": [
{
- "url": "https://1a-1791.com/video/s8/2/5/s/Q/8/5sQ8e.aaa.rec.mp4",
+ "url": "https://1a-1791.com/video/vo/47ww/m03gQsmi4GE/Px13ulRwHFM.mp4",
"filename": "CLOSING OUT Dave Rubin & Benny Johnson LIVE - Rumble Exclusive Live at TPU [v1at1kt][1080p].mp4",
"referer": "https://rumble.com/v1df7i1-rumble-exclusive-live-at-tpusa.html",
"album_id": None,
+ "download_folder": "re:RumbleEvents (Rumble)",
},
{
- "url": "https://1a-1791.com/video/s8/2/p/W/J/7/pWJ7e.aaa.rec.mp4",
+ "url": "https://1a-1791.com/video/vo/y7pe/dPdI3HbQ11Q/kkxZS05zoL0.mp4",
"filename": "Turning Point USA 2022 Student Action Summit DAY 1 - Rumble Exclusive Liv [v1amumr][1080p].mp4",
"referer": "https://rumble.com/v1d90jz-rumble-exclusive-live-at-tpusa.html",
},
{
- "url": "https://1a-1791.com/video/s8/2/b/f/f/8/bff8e.aaa.rec.mp4",
+ "url": "https://1a-1791.com/video/vo/5j7n/fsuRaNtWx5A/K9ZKZmzRXAg.mp4",
"filename": "President Donald J. Trump LIVE at Turning Point - Rumble Exclusive Live at [v1apryd][1080p].mp4",
"referer": "https://rumble.com/v1dbxvl-rumble-exclusive-live-at-tpusa.html",
},
diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.py
index d2adb5fc2..8a932d7e9 100644
--- a/tests/crawlers/test_crawlers.py
+++ b/tests/crawlers/test_crawlers.py
@@ -97,9 +97,9 @@ NOT_NONE = _NOT_NONE()
def _validate_results(crawler: Crawler, test_case: test_cases.CrawlerTestCase, results: list[MediaItem]) -> None:
- expected_results = sorted(test_case.results, key=lambda x: x["url"])
+ expected_results = dict(sorted(((x["url"], idx), x) for idx, x in enumerate(test_case.results, 1)))
origin = getattr(crawler, "PRIMARY_URL", AbsoluteHttpURL("https://google.com"))
- for index, (expected, media_item) in enumerate(zip(expected_results, results, strict=False), 1):
+ for (index, expected), media_item in zip(expected_results.items(), results, strict=False):
for attr_name, expected_value in expected.items():
result_value = getattr(media_item, attr_name)
if isinstance(result_value, Path):