Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: direct_link handle (HotPic) (#1083)
NTFSvolumefa97abf9df8e
diff --git a/cyberdrop_dl/crawlers/hotpic.py b/cyberdrop_dl/crawlers/hotpic.pyindex ba9e2cacf..2bb535e1f 100644--- a/cyberdrop_dl/crawlers/hotpic.py+++ b/cyberdrop_dl/crawlers/hotpic.py@@ -13,15 +13,23 @@ if TYPE_CHECKING:from cyberdrop_dl.data_structures.url_objects import ScrapeItem-IMAGE_SELECTOR = "img[id*=main-image]"-VIDEO_SELECTOR = "video > source"-ALBUM_ITEM_SELECTOR = "a[class*=spotlight]"++class Selectors:+ IMAGE = "img[id*=main-image]"+ VIDEO = "video > source"+ ALBUM_ITEM = "a[class*=spotlight]"+ IMAGE_OR_VIDEO = f"{IMAGE}, {VIDEO}"+PRIMARY_URL = AbsoluteHttpURL("https://hotpic.cc")+_SELECTORS = Selectors()class HotPicCrawler(Crawler):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Album": "/album/...", "Image": "/i/..."}+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Album": "/album/...",+ "Image": "/i/...",+ }SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "hotpic", "2385290.xyz"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLUPDATE_UNSUPPORTED: ClassVar[bool] = True@@ -46,7 +54,7 @@ class HotPicCrawler(Crawler):title = self.create_title(css.select_one_get_text(soup, "title").rsplit(" - ")[0], scrape_item.album_id)scrape_item.setup_as_profile(title, album_id=album_id)- for _, link in self.iter_tags(soup, ALBUM_ITEM_SELECTOR):+ for _, link in self.iter_tags(soup, _SELECTORS.ALBUM_ITEM):await self.handle_direct_link(scrape_item, link)@error_handling_wrapper@@ -57,7 +65,7 @@ class HotPicCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- file = soup.select_one(VIDEO_SELECTOR) or soup.select_one(IMAGE_SELECTOR)+ file = soup.select_one(_SELECTORS.IMAGE_OR_VIDEO)if not file:raise ScrapeError(422)link_str: str = css.get_attr(file, "src")@@ -65,8 +73,8 @@ class HotPicCrawler(Crawler):await self.handle_direct_link(scrape_item, link)@error_handling_wrapper- async def handle_direct_link(self, scrape_item: ScrapeItem) -> None:- link = thumbnail_to_img(scrape_item.url)+ async def handle_direct_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:+ link = thumbnail_to_img(link or scrape_item.url)canonical_url = link.with_host("hotpic.cc")filename, ext = self.get_filename_and_ext(canonical_url.name)await self.handle_file(canonical_url, scrape_item, filename, ext, debrid_link=link)