Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: crawler fixes (ashemaletube) (#914)

NTFSvolume1ccb7bd3bfda
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.py
index abb1b6acb..3e573b162 100644
--- a/cyberdrop_dl/crawlers/ashemaletube.py
+++ b/cyberdrop_dl/crawlers/ashemaletube.py
@@ -1,7 +1,5 @@
from __future__ import annotations
-import calendar
-import datetime
import json
import re
from typing import TYPE_CHECKING, NamedTuple
@@ -21,13 +19,17 @@ if TYPE_CHECKING:
from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
-PROFILE_VIDEOS_SELECTOR = "div.media-item__inner a[data-video-preview]"
-MODEL_VIDEO_SELECTOR = "a data-video-preview"
-USER_NAME_SELECTOR = "h1.username"
-PLAYLIST_VIDEOS_SELECTOR = "a.playlist-video-item__thumbnail"
-VIDEO_OBJECT_SCRIPT_SELECTOR = "script:contains('uploadDate')"
-JS_SELECTOR = "script:contains('var player = new VideoPlayer')"
-LOGIN_REQUIRED_SELECTOR = "div.loginLinks:contains('To watch this video please')"
+class Selectors:
+ PROFILE_VIDEOS = "div.media-item__inner a[data-video-preview]"
+ MODEL_VIDEO = "a data-video-preview"
+ USER_NAME = "h1.username"
+ PLAYLIST_VIDEOS = "a.playlist-video-item__thumbnail"
+ VIDEO_PROPS_JS = "script:contains('uploadDate')"
+ JS_PLAYER = "script:contains('var player = new VideoPlayer')"
+ LOGIN_REQUIRED = "div.loginLinks:contains('To watch this video please')"
+
+
+_SELECTORS = Selectors()
RESOLUTIONS = ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"] # best to worst
INCLUDE_VIDEO_ID_IN_FILENAME = True
@@ -39,7 +41,7 @@ class Format(NamedTuple):
class AShemaleTubeCrawler(Crawler):
primary_base_domain = URL("https://www.ashemaletube.com")
- next_page_selector = "a.rightKey"
+ next_page = "a.rightKey"
def __init__(self, manager: Manager) -> None:
super().__init__(manager, "ashemaletube", "aShemaleTube")
@@ -52,16 +54,14 @@ class AShemaleTubeCrawler(Crawler):
"""Determines where to send the scrape item based on the url."""
if any(p in scrape_item.url.parts for p in ("creators", "profiles", "pornstars", "model")):
return await self.model(scrape_item)
- elif "videos" in scrape_item.url.parts:
+ if "videos" in scrape_item.url.parts:
return await self.video(scrape_item)
- elif "playlists" in scrape_item.url.parts:
+ if "playlists" in scrape_item.url.parts:
return await self.playlist(scrape_item)
raise ValueError
@error_handling_wrapper
async def playlist(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup_cffi(self.domain, scrape_item.url)
title = ""
async for soup in self.web_pager(scrape_item.url, cffi=True):
if not title:
@@ -69,46 +69,44 @@ class AShemaleTubeCrawler(Crawler):
title = self.create_title(f"{playlist_name} [playlist]")
scrape_item.setup_as_album(title)
- for _, new_scrape_item in self.iter_children(scrape_item, soup, PLAYLIST_VIDEOS_SELECTOR):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.PLAYLIST_VIDEOS):
self.manager.task_group.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def model(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup_cffi(self.domain, scrape_item.url)
title = ""
async for soup in self.web_pager(scrape_item.url, cffi=True):
if not title:
- model_name = soup.select_one(USER_NAME_SELECTOR).get_text(strip=True) # type: ignore
+ model_name = soup.select_one(_SELECTORS.USER_NAME).get_text(strip=True) # type: ignore
title = self.create_title(f"{model_name} [model]")
scrape_item.setup_as_profile(title)
- for _, new_scrape_item in self.iter_children(scrape_item, soup, PROFILE_VIDEOS_SELECTOR):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.PROFILE_VIDEOS):
self.manager.task_group.create_task(self.run(new_scrape_item))
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
video_id: str = scrape_item.url.parts[2]
canonical_url = self.primary_base_domain / "videos" / video_id
- if await self.check_complete_from_referer(scrape_item.url):
+ if await self.check_complete_from_referer(canonical_url):
return
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup_cffi(self.domain, scrape_item.url)
- if soup.select_one(LOGIN_REQUIRED_SELECTOR):
+ if soup.select_one(_SELECTORS.LOGIN_REQUIRED):
raise ScrapeError(401)
- player = soup.select_one(JS_SELECTOR)
+ player = soup.select_one(_SELECTORS.JS_PLAYER)
if not player:
raise ScrapeError(422)
is_hls, best_format = parse_player_info(player.text)
if is_hls:
raise ScrapeError(422)
- if video_object := soup.select_one(VIDEO_OBJECT_SCRIPT_SELECTOR):
+ if video_object := soup.select_one(_SELECTORS.VIDEO_PROPS_JS):
json_data = json.loads(video_object.text.strip())
if "uploadDate" in json_data:
- scrape_item.possible_datetime = parse_datetime(json_data["uploadDate"])
+ scrape_item.possible_datetime = self.parse_date(json_data["uploadDate"])
title: str = soup.select_one("title").text.split("- aShemaletube.com")[0].strip() # type: ignore
link = self.parse_url(best_format.link_str)
@@ -147,9 +145,3 @@ def parse_player_info(script_text: str) -> tuple[bool, Format]:
format: Format = get_best_quality(json.loads(urls_info))
return is_hls, format
raise ScrapeError(404)
-
-
-def parse_datetime(date: str) -> int:
- """Parses a datetime string into a unix timestamp."""
- dt = datetime.datetime.fromisoformat(date)
- return calendar.timegm(dt.timetuple())