Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add gupload.xyz support (#48)

NTFSvolume9b4d356aee0a
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 3839a0139..517f5211c 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -47,6 +47,7 @@ from .girlsreleased import GirlsReleasedCrawler
from .gofile import GoFileCrawler
from .google_drive import GoogleDriveCrawler
from .google_photos import GooglePhotosCrawler
+from .gupload import GUploadCrawler
from .hclips import HClipsCrawler
from .hdzog import HDZogCrawler
from .hianime import HiAnimeCrawler
diff --git a/cyberdrop_dl/crawlers/gupload.py b/cyberdrop_dl/crawlers/gupload.py
new file mode 100644
index 000000000..f3958d36d
--- /dev/null
+++ b/cyberdrop_dl/crawlers/gupload.py
@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import base64
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Final
+
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.data_structures import Resolution
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, xor_decrypt
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+
+_KEY: Final = b"G7#kP!2qZxV9mRwL"
+
+
+def _decode_config(config_text: str) -> dict[str, Any]:
+ _, payload = config_text.split("~", 1)
+ config_bytes = base64.b64decode(payload)
+ return json.loads(xor_decrypt(config_bytes, _KEY))
+
+
+class GUploadCrawler(Crawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Video": ("/data/e/<video_id>",)}
+ DOMAIN: ClassVar[str] = "gupload"
+ FOLDER_DOMAIN: ClassVar[str] = "GUpload"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://gupload.xyz")
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["data", "e", video_id]:
+ return await self.video(scrape_item, video_id)
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
+ if await self.check_complete_from_referer(scrape_item):
+ return
+
+ config = await self._request_video_config(scrape_item.url)
+ m3u8_url = self.parse_url(config["videoUrl"])
+ m3u8 = await self.get_m3u8_from_index_url(m3u8_url)
+
+ filename, ext = self.get_filename_and_ext(video_id + ".mp4")
+ custom_filename = self.create_custom_filename(video_id, ext, resolution=Resolution.parse(m3u8_url))
+ await self.handle_file(m3u8_url, scrape_item, filename, ext, m3u8=m3u8, custom_filename=custom_filename)
+ thumbnail = self.parse_url(config["posterUrl"])
+ thumb_name, ext = self.get_filename_and_ext(f"{Path(custom_filename).stem}_thumb{thumbnail.suffix}")
+ await self.handle_file(
+ thumbnail,
+ scrape_item,
+ f"{video_id}_thumb{thumbnail.suffix}",
+ ext,
+ custom_filename=thumb_name,
+ referer=scrape_item.url.with_fragment("thumbnail"),
+ )
+
+ async def _request_video_config(self, url: AbsoluteHttpURL) -> dict[str, Any]:
+ html = await self.request_text(url, impersonate=True)
+ config_text = get_text_between(html, "var _cfg = ", "');").partition("('")[-1]
+ return _decode_config(config_text)
diff --git a/tests/crawlers/test_cases/gupload.py b/tests/crawlers/test_cases/gupload.py
new file mode 100644
index 000000000..06c729cb7
--- /dev/null
+++ b/tests/crawlers/test_cases/gupload.py
@@ -0,0 +1,26 @@
+DOMAIN = "gupload"
+TEST_CASES = [
+ (
+ "https://gupload.xyz/data/e/26011780d54f",
+ [
+ {
+ "url": "https://gupload.xyz/data/e/hls/26011780d54f/thumb/thumbnail_grid.jpg",
+ "referer": "https://gupload.xyz/data/e/26011780d54f#thumbnail",
+ "download_folder": "re:Loose Files (GUpload)",
+ "filename": "26011780d54f [720p]_thumb.jpg",
+ "original_filename": "26011780d54f_thumb.jpg",
+ "debrid_link": None,
+ "album_id": None,
+ },
+ {
+ "url": "https://gupload.xyz/data/e/hls/26011780d54f/720p.m3u8",
+ "referer": "https://gupload.xyz/data/e/26011780d54f",
+ "download_folder": "re:Loose Files (GUpload)",
+ "filename": "26011780d54f [720p].mp4",
+ "original_filename": "26011780d54f.mp4",
+ "debrid_link": None,
+ "album_id": None,
+ },
+ ],
+ ),
+]