Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add xasiat support (#1505)
NTFSvolume6c2adc615a30
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex 00a2d8606..c6b8d027e 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -118,6 +118,7 @@ from .vipr_dot_im import ViprImCrawlerfrom .voe_sx import VoeSxCrawlerfrom .wetransfer import WeTransferCrawlerfrom .wordpress import WP_CRAWLERS, WordPressHTMLCrawler, WordPressMediaCrawler+from .xasiat import XasiatCrawlerfrom .xbunkr import XBunkrCrawlerfrom .xenforo import XF_CRAWLERS, SimpCityCrawlerfrom .xgroovy import XGroovyCrawlerdiff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex 49d5f52cb..e97a35bd8 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -33,7 +33,7 @@ class Selectors:FLASHVARS = "script:-soup-contains('video_title:')"USER_NAME = "div.headline > h2"ALBUM_NAME = "div.headline > h1"- ALBUM_PICTURES = "div.album-list > a"+ ALBUM_PICTURES = "div.album-list > a, .images a"PICTURE = "div.photo-holder > img"PUBLIC_VIDEOS = "div#list_videos_public_videos_items"PRIVATE_VIDEOS = "div#list_videos_private_videos_items"@@ -135,10 +135,12 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):)@error_handling_wrapper- async def album(self, scrape_item: ScrapeItem) -> None:+ async def album(self, scrape_item: ScrapeItem, album_id: str | None = None) -> None:soup = await self.request_soup(scrape_item.url)- js_text = css.select_one_get_text(soup, _SELECTORS.ALBUM_ID)- album_id = get_text_between(js_text, "params['album_id'] =", ";").strip()+ if not album_id:+ js_text = css.select_one_get_text(soup, _SELECTORS.ALBUM_ID)+ album_id = get_text_between(js_text, "params['album_id'] =", ";")+results = await self.get_album_results(album_id)title = css.select_one_get_text(soup, _SELECTORS.ALBUM_NAME)title = self.create_title(f"{title} [album]", album_id)@@ -181,8 +183,9 @@ _find_flashvars = re.compile(r"(\w+):\s*'([^']*)'").findalldef _parse_video_vars(video_vars: str) -> KVSVideo:flashvars: dict[str, str] = dict(_find_flashvars(video_vars))- url_keys = filter(_match_video_url_keys, flashvars.keys())+ url_keys = list(filter(_match_video_url_keys, flashvars.keys()))license_token = _get_license_token(flashvars["license_code"])+ parse_resolution = Resolution.make_parser()def get_formats():for key in url_keys:@@ -190,7 +193,7 @@ def _parse_video_vars(video_vars: str) -> KVSVideo:if "/get_file/" not in url_str:continuequality = flashvars.get(f"{key}_text")- resolution = Resolution.highest() if quality == "HQ" else Resolution.parse(quality)+ resolution = Resolution.highest() if quality in ("HQ", "Best Quality") else parse_resolution(quality)url = _deobfuscate_url(url_str, license_token)yield resolution, urldiff --git a/cyberdrop_dl/crawlers/xasiat.py b/cyberdrop_dl/crawlers/xasiat.pynew file mode 100644index 000000000..276e27bf2--- /dev/null+++ b/cyberdrop_dl/crawlers/xasiat.py@@ -0,0 +1,32 @@+from __future__ import annotations++from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl.crawlers._kvs import KernelVideoSharingCrawler+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem++if TYPE_CHECKING:+ from cyberdrop_dl.crawlers.crawler import SupportedPaths+++class XasiatCrawler(KernelVideoSharingCrawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Album": "/albums/<id>/<name>",+ "Images": "/get_image/...",+ "Videos": "/videos/<id>/<name>",+ }++ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.xasiat.com")+ DOMAIN: ClassVar[str] = "xasiat"+ DEFAULT_TRIM_URLS: ClassVar[bool] = False++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["albums", album_id, _, *_]:+ return await self.album(scrape_item, album_id)+ case ["videos", _, _, *_]:+ return await self.video(scrape_item)+ case ["get_image", _, *_]:+ return await self.direct_file(scrape_item)+ case _:+ raise ValueErrordiff --git a/cyberdrop_dl/data_structures/mediaprops.py b/cyberdrop_dl/data_structures/mediaprops.pyindex 775ad15b8..5fde9562c 100644--- a/cyberdrop_dl/data_structures/mediaprops.py+++ b/cyberdrop_dl/data_structures/mediaprops.py@@ -4,8 +4,10 @@ import refrom fractions import Fractionfrom typing import TYPE_CHECKING, Final, NamedTuple+from cyberdrop_dl.exceptions import ScrapeError+if TYPE_CHECKING:- from collections.abc import Iterable+ from collections.abc import Callable, Iterableimport yarl@@ -114,6 +116,28 @@ class Resolution(NamedTuple):def highest() -> Resolution:return HIGHEST_RESOLUTION+ @staticmethod+ def make_parser() -> Callable[[yarl.URL | str | int | None], Resolution]:+ """Returns a callable wrapper around `Resolution.parse` that can return `Resolution.unknown()`++ Raises `ScrapeError` if more that 1 unknown resolution is parsed"""++ default_res: Resolution | None = None++ def parse(quality: yarl.URL | str | int | None, /) -> Resolution:+ nonlocal default_res+ try:+ return Resolution.parse(quality)+ except ValueError as e:+ if default_res is not None:+ msg = "Unable to select best quality. Resource has more that 1 unknown resolution"+ raise ScrapeError(422, msg) from e++ default_res = Resolution.unknown()+ return default_res++ return parse+UNKNOWN_RESOLUTION = Resolution.parse(0)HIGHEST_RESOLUTION = Resolution(9999, 9999)diff --git a/tests/crawlers/test_cases/xasiat.py b/tests/crawlers/test_cases/xasiat.pynew file mode 100644index 000000000..459674e81--- /dev/null+++ b/tests/crawlers/test_cases/xasiat.py@@ -0,0 +1,28 @@+DOMAIN = "xasiat"+TEST_CASES = [+ (+ "https://www.xasiat.com/albums/30487/cosplay-jk-61p-797mb",+ [+ {+ "url": "https://www.xasiat.com/get_image/2/01a28782c7940969a3a3d55ab1842268/sources/30000/30487/2155729.jpg/",+ "filename": "2155729.jpg",+ "referer": "https://www.xasiat.com/get_image/2/01a28782c7940969a3a3d55ab1842268/sources/30000/30487/2155729.jpg/",+ "album_id": "30487",+ "datetime": None,+ "download_folder": r"re:\[Cosplay\] 屿鱼 蔚蓝档案 妃咲 JK \[61P-797MB\] \[album\] \(Xasiat\)",+ }+ ],+ 61,+ ),+ (+ "https://www.xasiat.com/videos/104092/dvd-ai-shinozaki-special-bonus-dvd-weekly-playboy-2023-09-18-no-38/",+ [+ {+ "url": "ANY",+ "filename": "+付録DVD Ai Shinozaki 篠崎愛 – Special Bonus DVD Weekly Playboy 2023-09-18 No.38 [104092].mp4", # noqa: RUF001+ "original_filename": "104092_source.mp4",+ "album_id": None,+ }+ ],+ ),+]