Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: crawler fixes (bestprettygirls) (#915)
NTFSvolumedc16abffd6d6
diff --git a/cyberdrop_dl/crawlers/bestprettygirl.py b/cyberdrop_dl/crawlers/bestprettygirl.pyindex 08c73871e..715fb5e7f 100644--- a/cyberdrop_dl/crawlers/bestprettygirl.py+++ b/cyberdrop_dl/crawlers/bestprettygirl.py@@ -1,7 +1,5 @@from __future__ import annotations-import calendar-from datetime import datetimefrom typing import TYPE_CHECKINGfrom aiolimiter import AsyncLimiter@@ -18,17 +16,22 @@ if TYPE_CHECKING:from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem-IMAGES_SELECTOR = "div.elementor-widget-theme-post-content div.elementor-widget-container img"-VIDEO_IFRAME_SELECTOR = "div.elementor-widget-theme-post-content div.elementor-widget-container iframe"-TITLE_SELECTOR = "meta[property='og:title']"-DATE_SELECTOR = "meta[property='article:published_time']"+class Selectors:+ IMAGES = "div.elementor-widget-theme-post-content div.elementor-widget-container img"+ VIDEO_IFRAME = "div.elementor-widget-theme-post-content div.elementor-widget-container iframe"+ TITLE = "meta[property='og:title']"+ DATE = "meta[property='article:published_time']"+ COLLECTION_ITEM = "article.elementor-post.post a"+++_SELECTORS = Selectors()+COLLECTION_PARTS = "category", "tag", "date"-ITEM_SELECTOR = "article.elementor-post.post a"class BestPrettyGirlCrawler(Crawler):primary_base_domain = URL("https://bestprettygirl.com/")- next_page_selector = "a.page-numbers.next"+ next_page = "a.page-numbers.next"def __init__(self, manager: Manager) -> None:super().__init__(manager, "bestprettygirl.com", "BestPrettyGirl")@@ -49,7 +52,7 @@ class BestPrettyGirlCrawler(Crawler):collection_type = title = ""async for soup in self.web_pager(scrape_item.url):if not collection_type:- title_tag = soup.select_one(TITLE_SELECTOR)+ title_tag = soup.select_one(_SELECTORS.TITLE)if not title_tag:raise ScrapeError(422)@@ -66,12 +69,11 @@ class BestPrettyGirlCrawler(Crawler):else:title: str = title_tag.get_text(strip=True) # type: ignoretitle = title.removeprefix("Tag:").removeprefix("Category:").strip()- title = title + f" [{collection_type}]"- title = self.create_title(title)+ title = self.create_title(f"{title} [{collection_type}]")scrape_item.setup_as_album(title)- for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.COLLECTION_ITEM):self.manager.task_group.create_task(self.run(new_scrape_item))@error_handling_wrapper@@ -79,22 +81,22 @@ class BestPrettyGirlCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)- og_title: str = soup.select_one(TITLE_SELECTOR).get("content") # type: ignore- date_str: str = soup.select_one(DATE_SELECTOR).get("content") # type: ignore- date = datetime.fromisoformat(date_str)+ og_title: str = soup.select_one(_SELECTORS.TITLE)["content"] # type: ignore+ date_str: str = soup.select_one(_SELECTORS.DATE)["content"] # type: ignoretitle = self.create_title(og_title)scrape_item.setup_as_album(title)- scrape_item.possible_datetime = calendar.timegm(date.timetuple())+ scrape_item.possible_datetime = self.parse_date(date_str)- trash: str = ""- for _, link in self.iter_tags(soup, IMAGES_SELECTOR, "src"):- if not trash:- trash = link.name.split("-0000", 1)[0]+ trash_split = "-0000-"+ trash_len: int = 0+ for _, link in self.iter_tags(soup, _SELECTORS.IMAGES, "src"):+ if not trash_len:+ trash_len = link.name.find(trash_split) + len(trash_split)filename, ext = self.get_filename_and_ext(link.name)- custom_filename = link.name.replace(trash, "").removeprefix("-")+ custom_filename = link.name[trash_len:]custom_filename, _ = self.get_filename_and_ext(custom_filename)await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)- for _, new_scrape_item in self.iter_children(scrape_item, soup, VIDEO_IFRAME_SELECTOR, "data-src"):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEO_IFRAME, "data-src"):new_scrape_item.url = new_scrape_item.url.with_host("vidply.com")self.handle_external_links(new_scrape_item)