Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: Add MasaHub.com Support (#1456)
Jacob9b61545d6030
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex e474cacd1..63eb9dd05 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -58,6 +58,7 @@ from .jpg5 import JPG5Crawlerfrom .kemono import KemonoCrawlerfrom .leakedzone import LeakedZoneCrawlerfrom .luscious import LusciousCrawler+from .masahub import MasahubCrawlerfrom .mediafire import MediaFireCrawlerfrom .mega_nz import MegaNzCrawlerfrom .megacloud import MegaCloudCrawlerdiff --git a/cyberdrop_dl/crawlers/masahub.py b/cyberdrop_dl/crawlers/masahub.pynew file mode 100644index 000000000..08b0ea1e6--- /dev/null+++ b/cyberdrop_dl/crawlers/masahub.py@@ -0,0 +1,58 @@+from __future__ import annotations++from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import css+from cyberdrop_dl.utils.utilities import error_handling_wrapper++if TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem+++class Selector:+ VIDEO_SRC = css.CssAttributeSelector("a.video-btn, a.download-btn", "href")+ TITLE = "div.posts > h1 > strong, div.video_page_toolbar > h1"+ NEXT_PAGE = "div > a:-soup-contains('Next')"+ VIDEOS = "a.title, div.title > a"+++class MasahubCrawler(Crawler):+ SUPPORTED_DOMAINS = "masa49.com", "masahub.com", "masahub2.com", "masafun.net", "lol49.com", "vido99.com"+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Videos": "/title",+ "Search": "?s=<query>",+ }+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://masahub.com")+ DOMAIN: ClassVar[str] = "masahub.com"+ FOLDER_DOMAIN: ClassVar[str] = "Masahub"+ NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ if query := scrape_item.url.query.get("s"):+ return await self.search(scrape_item, query)+ elif len(scrape_item.url.parts) >= 2:+ return await self.video(scrape_item)+ raise ValueError++ @error_handling_wrapper+ async def video(self, scrape_item: ScrapeItem) -> None:+ if await self.check_complete_from_referer(scrape_item):+ return++ soup = await self.request_soup(scrape_item.url)+ link = self.parse_url(Selector.VIDEO_SRC(soup))+ title = css.select_one_get_text(soup, Selector.TITLE)+ _, ext = self.get_filename_and_ext(link.name)+ scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))+ custom_filename = self.create_custom_filename(title, ext)+ return await self.handle_file(link, scrape_item, link.name, ext, custom_filename=custom_filename)++ @error_handling_wrapper+ async def search(self, scrape_item: ScrapeItem, query: str) -> None:+ title = self.create_title(query)+ scrape_item.setup_as_album(title)+ async for soup in self.web_pager(scrape_item.url):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS):+ self.create_task(self.run(new_scrape_item))diff --git a/tests/crawlers/test_cases/masahub.py b/tests/crawlers/test_cases/masahub.pynew file mode 100644index 000000000..842b1eab7--- /dev/null+++ b/tests/crawlers/test_cases/masahub.py@@ -0,0 +1,21 @@+DOMAIN = "masahub.com"+TEST_CASES = [+ # Video+ (+ "https://masahub.com/desi-bbw-girl-nude-bathing-selfie-part-1/",+ [+ {+ "url": "https://old2.masahub.cc/files/5/232.mp4",+ "filename": "Desi bbw girl nude bathing selfie part 1.mp4",+ "referer": "https://masahub.com/desi-bbw-girl-nude-bathing-selfie-part-1/",+ "datetime": 1518384977,+ }+ ],+ ),+ # Search+ (+ "https://masahub.com/page/55/?s=bbw",+ [],+ 5,+ ),+]