Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: use new syntax (cyberdrop) (#923)

NTFSvolume8d5a82d65e07
diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.py
index e8e7182ee..c92fa1579 100644
--- a/cyberdrop_dl/crawlers/cyberdrop.py
+++ b/cyberdrop_dl/crawlers/cyberdrop.py
@@ -1,8 +1,5 @@
from __future__ import annotations
-import calendar
-import datetime
-import re
from typing import TYPE_CHECKING
from aiolimiter import AsyncLimiter
@@ -19,11 +16,16 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
-CDN_POSSIBILITIES = re.compile(r"^(?:(?:k1)[0-9]{0,2})(?:redir)?\.cyberdrop?\.[a-z]{2,3}$")
API_ENTRYPOINT = URL("https://api.cyberdrop.me/api/")
-ALBUM_TITLE_SELECTOR = "h1[id=title]"
-ALBUM_DATE_SELECTOR = "p[class=title]"
-ALBUM_ITEM_SELECTOR = "div[class*=image-container] a[class=image]"
+
+
+class Selectors:
+ ALBUM_TITLE = "h1[id=title]"
+ ALBUM_DATE = "p[class=title]"
+ ALBUM_ITEM = "div[class*=image-container] a[class=image]"
+
+
+_SELECTORS = Selectors()
class CyberdropCrawler(Crawler):
@@ -39,28 +41,30 @@ class CyberdropCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
"""Determines where to send the scrape item based on the url."""
if "a" in scrape_item.url.parts:
- scrape_item.url = scrape_item.url.with_query("nojs")
return await self.album(scrape_item)
await self.file(scrape_item)
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
"""Scrapes an album."""
+ scrape_item.url = scrape_item.url.with_query("nojs")
+ album_id = scrape_item.url.parts[2]
+
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)
- album_id = scrape_item.url.parts[2]
try:
- title = self.create_title(soup.select_one(ALBUM_TITLE_SELECTOR).text, album_id) # type: ignore
+ title: str = soup.select_one(_SELECTORS.ALBUM_TITLE).text # type: ignore
+ title = self.create_title(title, album_id)
scrape_item.setup_as_album(title, album_id=album_id)
except AttributeError:
msg = "Unable to parse album information from response content"
raise ScrapeError(422, msg) from None
- if date_tags := soup.select(ALBUM_DATE_SELECTOR):
- scrape_item.possible_datetime = parse_datetime(date_tags[-1].text)
+ if date_tags := soup.select(_SELECTORS.ALBUM_DATE):
+ scrape_item.possible_datetime = self.parse_date(date_tags[-1].text, "%d.%m.%Y")
- for _, new_scrape_item in self.iter_children(scrape_item, soup, ALBUM_ITEM_SELECTOR):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ALBUM_ITEM):
self.manager.task_group.create_task(self.run(new_scrape_item))
@error_handling_wrapper
@@ -70,14 +74,15 @@ class CyberdropCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item):
return
+ file_id = scrape_item.url.name
async with self.request_limiter:
- api_url = API_ENTRYPOINT / "file" / "info" / scrape_item.url.path[3:]
+ api_url = API_ENTRYPOINT / "file" / "info" / file_id
json_resp = await self.client.get_json(self.domain, api_url)
filename, ext = self.get_filename_and_ext(json_resp["name"])
async with self.request_limiter:
- api_url = API_ENTRYPOINT / "file" / "auth" / scrape_item.url.path[3:]
+ api_url = API_ENTRYPOINT / "file" / "auth" / file_id
json_resp = await self.client.get_json(self.domain, api_url)
link = self.parse_url(json_resp["url"])
@@ -89,21 +94,10 @@ class CyberdropCrawler(Crawler):
"""Gets the stream link for a given URL.
NOTE: This makes a request to get the final URL (if necessary). Calling function must use `@error_handling_wrapper`"""
+ assert url.host
if any(part in url.parts for part in ("a", "f")):
return url
- if is_cdn(url) or "e" in url.parts:
+ if url.host.count(".") > 1 or "e" in url.parts:
return self.primary_base_domain / "f" / url.name
_, streaming_url = await self.client.get_soup_and_return_url(self.domain, url)
return streaming_url
-
-
-def is_cdn(url: URL) -> bool:
- """Checks if a given URL is from a CDN."""
- assert url.host
- return bool(re.match(CDN_POSSIBILITIES, url.host))
-
-
-def parse_datetime(date: str) -> int:
- """Parses a datetime string into a unix timestamp."""
- parsed_date = datetime.datetime.strptime(date, "%d.%m.%Y")
- return calendar.timegm(parsed_date.timetuple())