Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: crawler fixes (ashemaletube) (#914)
NTFSvolume1ccb7bd3bfda
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.pyindex abb1b6acb..3e573b162 100644--- a/cyberdrop_dl/crawlers/ashemaletube.py+++ b/cyberdrop_dl/crawlers/ashemaletube.py@@ -1,7 +1,5 @@from __future__ import annotations-import calendar-import datetimeimport jsonimport refrom typing import TYPE_CHECKING, NamedTuple@@ -21,13 +19,17 @@ if TYPE_CHECKING:from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem-PROFILE_VIDEOS_SELECTOR = "div.media-item__inner a[data-video-preview]"-MODEL_VIDEO_SELECTOR = "a data-video-preview"-USER_NAME_SELECTOR = "h1.username"-PLAYLIST_VIDEOS_SELECTOR = "a.playlist-video-item__thumbnail"-VIDEO_OBJECT_SCRIPT_SELECTOR = "script:contains('uploadDate')"-JS_SELECTOR = "script:contains('var player = new VideoPlayer')"-LOGIN_REQUIRED_SELECTOR = "div.loginLinks:contains('To watch this video please')"+class Selectors:+ PROFILE_VIDEOS = "div.media-item__inner a[data-video-preview]"+ MODEL_VIDEO = "a data-video-preview"+ USER_NAME = "h1.username"+ PLAYLIST_VIDEOS = "a.playlist-video-item__thumbnail"+ VIDEO_PROPS_JS = "script:contains('uploadDate')"+ JS_PLAYER = "script:contains('var player = new VideoPlayer')"+ LOGIN_REQUIRED = "div.loginLinks:contains('To watch this video please')"+++_SELECTORS = Selectors()RESOLUTIONS = ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"] # best to worstINCLUDE_VIDEO_ID_IN_FILENAME = True@@ -39,7 +41,7 @@ class Format(NamedTuple):class AShemaleTubeCrawler(Crawler):primary_base_domain = URL("https://www.ashemaletube.com")- next_page_selector = "a.rightKey"+ next_page = "a.rightKey"def __init__(self, manager: Manager) -> None:super().__init__(manager, "ashemaletube", "aShemaleTube")@@ -52,16 +54,14 @@ class AShemaleTubeCrawler(Crawler):"""Determines where to send the scrape item based on the url."""if any(p in scrape_item.url.parts for p in ("creators", "profiles", "pornstars", "model")):return await self.model(scrape_item)- elif "videos" in scrape_item.url.parts:+ if "videos" in scrape_item.url.parts:return await self.video(scrape_item)- elif "playlists" in scrape_item.url.parts:+ if "playlists" in scrape_item.url.parts:return await self.playlist(scrape_item)raise ValueError@error_handling_wrapperasync def playlist(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup_cffi(self.domain, scrape_item.url)title = ""async for soup in self.web_pager(scrape_item.url, cffi=True):if not title:@@ -69,46 +69,44 @@ class AShemaleTubeCrawler(Crawler):title = self.create_title(f"{playlist_name} [playlist]")scrape_item.setup_as_album(title)- for _, new_scrape_item in self.iter_children(scrape_item, soup, PLAYLIST_VIDEOS_SELECTOR):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.PLAYLIST_VIDEOS):self.manager.task_group.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def model(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup_cffi(self.domain, scrape_item.url)title = ""async for soup in self.web_pager(scrape_item.url, cffi=True):if not title:- model_name = soup.select_one(USER_NAME_SELECTOR).get_text(strip=True) # type: ignore+ model_name = soup.select_one(_SELECTORS.USER_NAME).get_text(strip=True) # type: ignoretitle = self.create_title(f"{model_name} [model]")scrape_item.setup_as_profile(title)- for _, new_scrape_item in self.iter_children(scrape_item, soup, PROFILE_VIDEOS_SELECTOR):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.PROFILE_VIDEOS):self.manager.task_group.create_task(self.run(new_scrape_item))@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:video_id: str = scrape_item.url.parts[2]canonical_url = self.primary_base_domain / "videos" / video_id- if await self.check_complete_from_referer(scrape_item.url):+ if await self.check_complete_from_referer(canonical_url):returnasync with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup_cffi(self.domain, scrape_item.url)- if soup.select_one(LOGIN_REQUIRED_SELECTOR):+ if soup.select_one(_SELECTORS.LOGIN_REQUIRED):raise ScrapeError(401)- player = soup.select_one(JS_SELECTOR)+ player = soup.select_one(_SELECTORS.JS_PLAYER)if not player:raise ScrapeError(422)is_hls, best_format = parse_player_info(player.text)if is_hls:raise ScrapeError(422)- if video_object := soup.select_one(VIDEO_OBJECT_SCRIPT_SELECTOR):+ if video_object := soup.select_one(_SELECTORS.VIDEO_PROPS_JS):json_data = json.loads(video_object.text.strip())if "uploadDate" in json_data:- scrape_item.possible_datetime = parse_datetime(json_data["uploadDate"])+ scrape_item.possible_datetime = self.parse_date(json_data["uploadDate"])title: str = soup.select_one("title").text.split("- aShemaletube.com")[0].strip() # type: ignorelink = self.parse_url(best_format.link_str)@@ -147,9 +145,3 @@ def parse_player_info(script_text: str) -> tuple[bool, Format]:format: Format = get_best_quality(json.loads(urls_info))return is_hls, formatraise ScrapeError(404)---def parse_datetime(date: str) -> int:- """Parses a datetime string into a unix timestamp."""- dt = datetime.datetime.fromisoformat(date)- return calendar.timegm(dt.timetuple())