Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add xasiat support (#1505)

NTFSvolume6c2adc615a30
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 00a2d8606..c6b8d027e 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -118,6 +118,7 @@ from .vipr_dot_im import ViprImCrawler
from .voe_sx import VoeSxCrawler
from .wetransfer import WeTransferCrawler
from .wordpress import WP_CRAWLERS, WordPressHTMLCrawler, WordPressMediaCrawler
+from .xasiat import XasiatCrawler
from .xbunkr import XBunkrCrawler
from .xenforo import XF_CRAWLERS, SimpCityCrawler
from .xgroovy import XGroovyCrawler
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.py
index 49d5f52cb..e97a35bd8 100644
--- a/cyberdrop_dl/crawlers/_kvs.py
+++ b/cyberdrop_dl/crawlers/_kvs.py
@@ -33,7 +33,7 @@ class Selectors:
FLASHVARS = "script:-soup-contains('video_title:')"
USER_NAME = "div.headline > h2"
ALBUM_NAME = "div.headline > h1"
- ALBUM_PICTURES = "div.album-list > a"
+ ALBUM_PICTURES = "div.album-list > a, .images a"
PICTURE = "div.photo-holder > img"
PUBLIC_VIDEOS = "div#list_videos_public_videos_items"
PRIVATE_VIDEOS = "div#list_videos_private_videos_items"
@@ -135,10 +135,12 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
)
@error_handling_wrapper
- async def album(self, scrape_item: ScrapeItem) -> None:
+ async def album(self, scrape_item: ScrapeItem, album_id: str | None = None) -> None:
soup = await self.request_soup(scrape_item.url)
- js_text = css.select_one_get_text(soup, _SELECTORS.ALBUM_ID)
- album_id = get_text_between(js_text, "params['album_id'] =", ";").strip()
+ if not album_id:
+ js_text = css.select_one_get_text(soup, _SELECTORS.ALBUM_ID)
+ album_id = get_text_between(js_text, "params['album_id'] =", ";")
+
results = await self.get_album_results(album_id)
title = css.select_one_get_text(soup, _SELECTORS.ALBUM_NAME)
title = self.create_title(f"{title} [album]", album_id)
@@ -181,8 +183,9 @@ _find_flashvars = re.compile(r"(\w+):\s*'([^']*)'").findall
def _parse_video_vars(video_vars: str) -> KVSVideo:
flashvars: dict[str, str] = dict(_find_flashvars(video_vars))
- url_keys = filter(_match_video_url_keys, flashvars.keys())
+ url_keys = list(filter(_match_video_url_keys, flashvars.keys()))
license_token = _get_license_token(flashvars["license_code"])
+ parse_resolution = Resolution.make_parser()
def get_formats():
for key in url_keys:
@@ -190,7 +193,7 @@ def _parse_video_vars(video_vars: str) -> KVSVideo:
if "/get_file/" not in url_str:
continue
quality = flashvars.get(f"{key}_text")
- resolution = Resolution.highest() if quality == "HQ" else Resolution.parse(quality)
+ resolution = Resolution.highest() if quality in ("HQ", "Best Quality") else parse_resolution(quality)
url = _deobfuscate_url(url_str, license_token)
yield resolution, url
diff --git a/cyberdrop_dl/crawlers/xasiat.py b/cyberdrop_dl/crawlers/xasiat.py
new file mode 100644
index 000000000..276e27bf2
--- /dev/null
+++ b/cyberdrop_dl/crawlers/xasiat.py
@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.crawlers.crawler import SupportedPaths
+
+
+class XasiatCrawler(KernelVideoSharingCrawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Album": "/albums/<id>/<name>",
+ "Images": "/get_image/...",
+ "Videos": "/videos/<id>/<name>",
+ }
+
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.xasiat.com")
+ DOMAIN: ClassVar[str] = "xasiat"
+ DEFAULT_TRIM_URLS: ClassVar[bool] = False
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["albums", album_id, _, *_]:
+ return await self.album(scrape_item, album_id)
+ case ["videos", _, _, *_]:
+ return await self.video(scrape_item)
+ case ["get_image", _, *_]:
+ return await self.direct_file(scrape_item)
+ case _:
+ raise ValueError
diff --git a/cyberdrop_dl/data_structures/mediaprops.py b/cyberdrop_dl/data_structures/mediaprops.py
index 775ad15b8..5fde9562c 100644
--- a/cyberdrop_dl/data_structures/mediaprops.py
+++ b/cyberdrop_dl/data_structures/mediaprops.py
@@ -4,8 +4,10 @@ import re
from fractions import Fraction
from typing import TYPE_CHECKING, Final, NamedTuple
+from cyberdrop_dl.exceptions import ScrapeError
+
if TYPE_CHECKING:
- from collections.abc import Iterable
+ from collections.abc import Callable, Iterable
import yarl
@@ -114,6 +116,28 @@ class Resolution(NamedTuple):
def highest() -> Resolution:
return HIGHEST_RESOLUTION
+ @staticmethod
+ def make_parser() -> Callable[[yarl.URL | str | int | None], Resolution]:
+ """Returns a callable wrapper around `Resolution.parse` that can return `Resolution.unknown()`
+
+ Raises `ScrapeError` if more that 1 unknown resolution is parsed"""
+
+ default_res: Resolution | None = None
+
+ def parse(quality: yarl.URL | str | int | None, /) -> Resolution:
+ nonlocal default_res
+ try:
+ return Resolution.parse(quality)
+ except ValueError as e:
+ if default_res is not None:
+ msg = "Unable to select best quality. Resource has more that 1 unknown resolution"
+ raise ScrapeError(422, msg) from e
+
+ default_res = Resolution.unknown()
+ return default_res
+
+ return parse
+
UNKNOWN_RESOLUTION = Resolution.parse(0)
HIGHEST_RESOLUTION = Resolution(9999, 9999)
diff --git a/tests/crawlers/test_cases/xasiat.py b/tests/crawlers/test_cases/xasiat.py
new file mode 100644
index 000000000..459674e81
--- /dev/null
+++ b/tests/crawlers/test_cases/xasiat.py
@@ -0,0 +1,28 @@
+DOMAIN = "xasiat"
+TEST_CASES = [
+ (
+ "https://www.xasiat.com/albums/30487/cosplay-jk-61p-797mb",
+ [
+ {
+ "url": "https://www.xasiat.com/get_image/2/01a28782c7940969a3a3d55ab1842268/sources/30000/30487/2155729.jpg/",
+ "filename": "2155729.jpg",
+ "referer": "https://www.xasiat.com/get_image/2/01a28782c7940969a3a3d55ab1842268/sources/30000/30487/2155729.jpg/",
+ "album_id": "30487",
+ "datetime": None,
+ "download_folder": r"re:\[Cosplay\] 屿鱼 蔚蓝档案 妃咲 JK \[61P-797MB\] \[album\] \(Xasiat\)",
+ }
+ ],
+ 61,
+ ),
+ (
+ "https://www.xasiat.com/videos/104092/dvd-ai-shinozaki-special-bonus-dvd-weekly-playboy-2023-09-18-no-38/",
+ [
+ {
+ "url": "ANY",
+ "filename": "+付録DVD Ai Shinozaki 篠崎愛 – Special Bonus DVD Weekly Playboy 2023-09-18 No.38 [104092].mp4", # noqa: RUF001
+ "original_filename": "104092_source.mp4",
+ "album_id": None,
+ }
+ ],
+ ),
+]