Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add gupload.xyz support (#48)
NTFSvolume9b4d356aee0a
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex 3839a0139..517f5211c 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -47,6 +47,7 @@ from .girlsreleased import GirlsReleasedCrawlerfrom .gofile import GoFileCrawlerfrom .google_drive import GoogleDriveCrawlerfrom .google_photos import GooglePhotosCrawler+from .gupload import GUploadCrawlerfrom .hclips import HClipsCrawlerfrom .hdzog import HDZogCrawlerfrom .hianime import HiAnimeCrawlerdiff --git a/cyberdrop_dl/crawlers/gupload.py b/cyberdrop_dl/crawlers/gupload.pynew file mode 100644index 000000000..f3958d36d--- /dev/null+++ b/cyberdrop_dl/crawlers/gupload.py@@ -0,0 +1,65 @@+from __future__ import annotations++import base64+import json+from pathlib import Path+from typing import TYPE_CHECKING, Any, ClassVar, Final++from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.data_structures import Resolution+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, xor_decrypt++if TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem+++_KEY: Final = b"G7#kP!2qZxV9mRwL"+++def _decode_config(config_text: str) -> dict[str, Any]:+ _, payload = config_text.split("~", 1)+ config_bytes = base64.b64decode(payload)+ return json.loads(xor_decrypt(config_bytes, _KEY))+++class GUploadCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Video": ("/data/e/<video_id>",)}+ DOMAIN: ClassVar[str] = "gupload"+ FOLDER_DOMAIN: ClassVar[str] = "GUpload"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://gupload.xyz")++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["data", "e", video_id]:+ return await self.video(scrape_item, video_id)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:+ if await self.check_complete_from_referer(scrape_item):+ return++ config = await self._request_video_config(scrape_item.url)+ m3u8_url = self.parse_url(config["videoUrl"])+ m3u8 = await self.get_m3u8_from_index_url(m3u8_url)++ filename, ext = self.get_filename_and_ext(video_id + ".mp4")+ custom_filename = self.create_custom_filename(video_id, ext, resolution=Resolution.parse(m3u8_url))+ await self.handle_file(m3u8_url, scrape_item, filename, ext, m3u8=m3u8, custom_filename=custom_filename)+ thumbnail = self.parse_url(config["posterUrl"])+ thumb_name, ext = self.get_filename_and_ext(f"{Path(custom_filename).stem}_thumb{thumbnail.suffix}")+ await self.handle_file(+ thumbnail,+ scrape_item,+ f"{video_id}_thumb{thumbnail.suffix}",+ ext,+ custom_filename=thumb_name,+ referer=scrape_item.url.with_fragment("thumbnail"),+ )++ async def _request_video_config(self, url: AbsoluteHttpURL) -> dict[str, Any]:+ html = await self.request_text(url, impersonate=True)+ config_text = get_text_between(html, "var _cfg = ", "');").partition("('")[-1]+ return _decode_config(config_text)diff --git a/tests/crawlers/test_cases/gupload.py b/tests/crawlers/test_cases/gupload.pynew file mode 100644index 000000000..06c729cb7--- /dev/null+++ b/tests/crawlers/test_cases/gupload.py@@ -0,0 +1,26 @@+DOMAIN = "gupload"+TEST_CASES = [+ (+ "https://gupload.xyz/data/e/26011780d54f",+ [+ {+ "url": "https://gupload.xyz/data/e/hls/26011780d54f/thumb/thumbnail_grid.jpg",+ "referer": "https://gupload.xyz/data/e/26011780d54f#thumbnail",+ "download_folder": "re:Loose Files (GUpload)",+ "filename": "26011780d54f [720p]_thumb.jpg",+ "original_filename": "26011780d54f_thumb.jpg",+ "debrid_link": None,+ "album_id": None,+ },+ {+ "url": "https://gupload.xyz/data/e/hls/26011780d54f/720p.m3u8",+ "referer": "https://gupload.xyz/data/e/26011780d54f",+ "download_folder": "re:Loose Files (GUpload)",+ "filename": "26011780d54f [720p].mp4",+ "original_filename": "26011780d54f.mp4",+ "debrid_link": None,+ "album_id": None,+ },+ ],+ ),+]