Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: Add MasaHub.com Support (#1456)

Jacob9b61545d6030
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index e474cacd1..63eb9dd05 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -58,6 +58,7 @@ from .jpg5 import JPG5Crawler
from .kemono import KemonoCrawler
from .leakedzone import LeakedZoneCrawler
from .luscious import LusciousCrawler
+from .masahub import MasahubCrawler
from .mediafire import MediaFireCrawler
from .mega_nz import MegaNzCrawler
from .megacloud import MegaCloudCrawler
diff --git a/cyberdrop_dl/crawlers/masahub.py b/cyberdrop_dl/crawlers/masahub.py
new file mode 100644
index 000000000..08b0ea1e6
--- /dev/null
+++ b/cyberdrop_dl/crawlers/masahub.py
@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import css
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+
+class Selector:
+ VIDEO_SRC = css.CssAttributeSelector("a.video-btn, a.download-btn", "href")
+ TITLE = "div.posts > h1 > strong, div.video_page_toolbar > h1"
+ NEXT_PAGE = "div > a:-soup-contains('Next')"
+ VIDEOS = "a.title, div.title > a"
+
+
+class MasahubCrawler(Crawler):
+ SUPPORTED_DOMAINS = "masa49.com", "masahub.com", "masahub2.com", "masafun.net", "lol49.com", "vido99.com"
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Videos": "/title",
+ "Search": "?s=<query>",
+ }
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://masahub.com")
+ DOMAIN: ClassVar[str] = "masahub.com"
+ FOLDER_DOMAIN: ClassVar[str] = "Masahub"
+ NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ if query := scrape_item.url.query.get("s"):
+ return await self.search(scrape_item, query)
+ elif len(scrape_item.url.parts) >= 2:
+ return await self.video(scrape_item)
+ raise ValueError
+
+ @error_handling_wrapper
+ async def video(self, scrape_item: ScrapeItem) -> None:
+ if await self.check_complete_from_referer(scrape_item):
+ return
+
+ soup = await self.request_soup(scrape_item.url)
+ link = self.parse_url(Selector.VIDEO_SRC(soup))
+ title = css.select_one_get_text(soup, Selector.TITLE)
+ _, ext = self.get_filename_and_ext(link.name)
+ scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))
+ custom_filename = self.create_custom_filename(title, ext)
+ return await self.handle_file(link, scrape_item, link.name, ext, custom_filename=custom_filename)
+
+ @error_handling_wrapper
+ async def search(self, scrape_item: ScrapeItem, query: str) -> None:
+ title = self.create_title(query)
+ scrape_item.setup_as_album(title)
+ async for soup in self.web_pager(scrape_item.url):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS):
+ self.create_task(self.run(new_scrape_item))
diff --git a/tests/crawlers/test_cases/masahub.py b/tests/crawlers/test_cases/masahub.py
new file mode 100644
index 000000000..842b1eab7
--- /dev/null
+++ b/tests/crawlers/test_cases/masahub.py
@@ -0,0 +1,21 @@
+DOMAIN = "masahub.com"
+TEST_CASES = [
+ # Video
+ (
+ "https://masahub.com/desi-bbw-girl-nude-bathing-selfie-part-1/",
+ [
+ {
+ "url": "https://old2.masahub.cc/files/5/232.mp4",
+ "filename": "Desi bbw girl nude bathing selfie part 1.mp4",
+ "referer": "https://masahub.com/desi-bbw-girl-nude-bathing-selfie-part-1/",
+ "datetime": 1518384977,
+ }
+ ],
+ ),
+ # Search
+ (
+ "https://masahub.com/page/55/?s=bbw",
+ [],
+ 5,
+ ),
+]