Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: use new syntax (cyberdrop) (#923)
NTFSvolume8d5a82d65e07
diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.pyindex e8e7182ee..c92fa1579 100644--- a/cyberdrop_dl/crawlers/cyberdrop.py+++ b/cyberdrop_dl/crawlers/cyberdrop.py@@ -1,8 +1,5 @@from __future__ import annotations-import calendar-import datetime-import refrom typing import TYPE_CHECKINGfrom aiolimiter import AsyncLimiter@@ -19,11 +16,16 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem-CDN_POSSIBILITIES = re.compile(r"^(?:(?:k1)[0-9]{0,2})(?:redir)?\.cyberdrop?\.[a-z]{2,3}$")API_ENTRYPOINT = URL("https://api.cyberdrop.me/api/")-ALBUM_TITLE_SELECTOR = "h1[id=title]"-ALBUM_DATE_SELECTOR = "p[class=title]"-ALBUM_ITEM_SELECTOR = "div[class*=image-container] a[class=image]"+++class Selectors:+ ALBUM_TITLE = "h1[id=title]"+ ALBUM_DATE = "p[class=title]"+ ALBUM_ITEM = "div[class*=image-container] a[class=image]"+++_SELECTORS = Selectors()class CyberdropCrawler(Crawler):@@ -39,28 +41,30 @@ class CyberdropCrawler(Crawler):async def fetch(self, scrape_item: ScrapeItem) -> None:"""Determines where to send the scrape item based on the url."""if "a" in scrape_item.url.parts:- scrape_item.url = scrape_item.url.with_query("nojs")return await self.album(scrape_item)await self.file(scrape_item)@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:"""Scrapes an album."""+ scrape_item.url = scrape_item.url.with_query("nojs")+ album_id = scrape_item.url.parts[2]+async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)- album_id = scrape_item.url.parts[2]try:- title = self.create_title(soup.select_one(ALBUM_TITLE_SELECTOR).text, album_id) # type: ignore+ title: str = soup.select_one(_SELECTORS.ALBUM_TITLE).text # type: ignore+ title = self.create_title(title, album_id)scrape_item.setup_as_album(title, album_id=album_id)except AttributeError:msg = "Unable to parse album information from response content"raise ScrapeError(422, msg) from None- if date_tags := soup.select(ALBUM_DATE_SELECTOR):- scrape_item.possible_datetime = parse_datetime(date_tags[-1].text)+ if date_tags := soup.select(_SELECTORS.ALBUM_DATE):+ scrape_item.possible_datetime = self.parse_date(date_tags[-1].text, "%d.%m.%Y")- for _, new_scrape_item in self.iter_children(scrape_item, soup, ALBUM_ITEM_SELECTOR):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUM_ITEM):self.manager.task_group.create_task(self.run(new_scrape_item))@error_handling_wrapper@@ -70,14 +74,15 @@ class CyberdropCrawler(Crawler):if await self.check_complete_from_referer(scrape_item):return+ file_id = scrape_item.url.nameasync with self.request_limiter:- api_url = API_ENTRYPOINT / "file" / "info" / scrape_item.url.path[3:]+ api_url = API_ENTRYPOINT / "file" / "info" / file_idjson_resp = await self.client.get_json(self.domain, api_url)filename, ext = self.get_filename_and_ext(json_resp["name"])async with self.request_limiter:- api_url = API_ENTRYPOINT / "file" / "auth" / scrape_item.url.path[3:]+ api_url = API_ENTRYPOINT / "file" / "auth" / file_idjson_resp = await self.client.get_json(self.domain, api_url)link = self.parse_url(json_resp["url"])@@ -89,21 +94,10 @@ class CyberdropCrawler(Crawler):"""Gets the stream link for a given URL.NOTE: This makes a request to get the final URL (if necessary). Calling function must use `@error_handling_wrapper`"""+ assert url.hostif any(part in url.parts for part in ("a", "f")):return url- if is_cdn(url) or "e" in url.parts:+ if url.host.count(".") > 1 or "e" in url.parts:return self.primary_base_domain / "f" / url.name_, streaming_url = await self.client.get_soup_and_return_url(self.domain, url)return streaming_url---def is_cdn(url: URL) -> bool:- """Checks if a given URL is from a CDN."""- assert url.host- return bool(re.match(CDN_POSSIBILITIES, url.host))---def parse_datetime(date: str) -> int:- """Parses a datetime string into a unix timestamp."""- parsed_date = datetime.datetime.strptime(date, "%d.%m.%Y")- return calendar.timegm(parsed_date.timetuple())