Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: rework crawler (Motherless) (#2028)
NTFSvolumeff4726e00406
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c887f44af..8ad008490 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Dead lock if more that 50 downloads are queued (Filester)- Solve proof of work challenge (Fileditch)+- User downloads (Motherless)## [10.2.0] - 2026-07-13diff --git a/cyberdrop_dl/crawlers/motherless.py b/cyberdrop_dl/crawlers/motherless.pyindex b9f15afba..6c68acf06 100644--- a/cyberdrop_dl/crawlers/motherless.py+++ b/cyberdrop_dl/crawlers/motherless.py@@ -1,214 +1,224 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple+import dataclasses+from pathlib import Path+from typing import TYPE_CHECKING, ClassVar, final, overridefrom cyberdrop_dl import aio+from cyberdrop_dl.crawlers import Registryfrom cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css+from cyberdrop_dl.utils import TextExtractor, css, dates, parse_urlfrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:+ import datetime+ from collections.abc import AsyncIterable, Generator+from bs4 import BeautifulSoupfrom cyberdrop_dl.url_objects import ScrapeItem-PRIMARY_URL = AbsoluteHttpURL("https://motherless.xxx")-MEDIA_INFO_JS_SELECTOR = "script:-soup-contains('__fileurl')"-ITEM_SELECTOR = "div.thumb-container a.img-container"-ITEM_TITLE_SELECTOR = "div.media-meta-title"-GALLERY_TITLE_SELECTOR = "div.gallery-title > h2"-GROUP_TITLE_SELECTOR = "div.group-bio > h1"-ITEM_GALLERY_TITLE_SELECTOR = "div.gallery-captions > a.gallery-data"-NOT_FOUND_TEXTS = "The page you're looking for cannot be found", "File not Found. Nothing to see here"-USER_NAME_SELECTOR = "div.member-bio-username"---class MediaInfo(NamedTuple):- type: str- url: str+@final+class Selector:+ MEDIA_INFO_JS = "script:-soup-contains('__fileurl')"+ THUMB = ".thumb-container [data-codename]"+ COLLECTION_TITLE = ".gallery-title > h2, .group-bio > h1"+ USER_NAME = "div.member-bio-username"[email protected]_refererclass MotherlessCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Group": ("/g/<group_name>", "/gi/<image>", "/gv/<video>"),- "User": ("/u/...", "/f/..."),- "Image": "/...",- "Video": "pending",- "**NOTE**": "Galleries are NOT supported",+ "Group": (+ "/g/<group_name>",+ "/gi/<group_name>",+ "/gv/<group_name>",+ ),+ "Gallery": (+ "/G<gallery_id>",+ "/GI<gallery_id>",+ "/GV<gallery_id>",+ ),+ "User": (+ "/m/<user_name>",+ "/member/<user_name>",+ "/u/<user_name>",+ "/u/<user_name>?t=v",+ "/u/<user_name>?t=i",+ ),+ "User galleries": "/galleries/member/<user_name>/...",+ "Image or Video": (+ "/<media_id>",+ "/g/<group_name>/<media_id>",+ "/G<gallery_id>/<media_id>",+ ),}- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL- NEXT_PAGE_SELECTOR: ClassVar[str] = "div.pagination_link > a[rel=next]"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://motherless.xxx")+ NEXT_PAGE_SELECTOR: ClassVar[str] = ".pagination_link > a[rel=next]"DOMAIN: ClassVar[str] = "motherless"OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("motherless.com",)_RATE_LIMIT: ClassVar[RateLimit] = 2, 1- async def fetch(self, scrape_item: ScrapeItem, collection_id: str = "") -> None:- parts = scrape_item.url.parts- n_parts = len(parts)- item_id = collection_id or scrape_item.url.name- is_gallery_homepage = scrape_item.url.name.startswith("G") and n_parts == 2- is_group_homepage = "g" in parts and n_parts == 3- is_user = any(p in parts for p in ("u", "f")) # /member/ is for user galleries, not supported yet- is_videos_or_images = any(p in parts for p in ("videos", "images"))- is_supported_user_url = is_user and (n_parts == 3 or (n_parts > 3 and is_videos_or_images))-- if "gi" in parts: # group images- return await self.collection_items(scrape_item, "images", item_id)- if "gv" in parts: # group videos- return await self.collection_items(scrape_item, "videos", item_id)- if is_group_homepage or is_gallery_homepage: # gallery or group- return await self.collection(scrape_item)- if is_supported_user_url:- return await self.user(scrape_item)- if is_user:- raise ValueError- return await self.media(scrape_item)+ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["g", slug]:+ await self.group(scrape_item, slug)+ case ["gv" | "gi" as prefix, slug]:+ name = "images" if prefix == "gi" else "videos"+ await self.collection(scrape_item, slug, name)++ case [slug] if slug.startswith("G") and (gallery_id := slug[2:]):+ if (prefix := slug[:2]) in ("GV", "GI"):+ name = "images" if prefix == "GI" else "videos"+ return await self.collection(scrape_item, gallery_id, name)++ gallery_id = slug[1:]+ await self.gallery(scrape_item, gallery_id)++ case [media_id]:+ await self.media(scrape_item, media_id)++ case ["u", username]:+ type_ = scrape_item.url.query.get("t")+ if type_ not in ("v", "i"):+ return await self.user(scrape_item, username)+ name = "images" if type_ == "i" else "videos"+ await self.user_collection(scrape_item, username, name)++ case ["galleries", "member", username, *_]:+ await self.user_collection(scrape_item, username, "galleries")++ case _:+ raise ValueError++ @classmethod+ @override+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ url = super().transform_url(url)+ match url.parts[1:]:+ case [slug, media_id] if slug.startswith("G") and slug[1:]:+ return url.origin() / media_id+ case ["g", _, media_id]:+ return url.origin() / media_id+ case ["member" | "m", username]:+ return url.origin() / "u" / username+ case _:+ return url@error_handling_wrapper- async def user(self, scrape_item: ScrapeItem) -> None:- n_parts = len(scrape_item.url.parts)- assert n_parts >= 3- username = scrape_item.url.parts[2]- canonical_url = PRIMARY_URL / "f" / username- videos_url = canonical_url / "videos"- images_url = canonical_url / "images"- is_homepage = n_parts == 3-- title: str = f"{username} [user]"- title = self.create_title(title)- scrape_item.setup_as_album(title)-- if is_homepage or "images" in scrape_item.url.parts:- async for soup in self.web_pager(images_url):- check_soup(soup)- for new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):- new_scrape_item.append_folders("Images")- self.create_task(self.run(new_scrape_item))-- if is_homepage or "videos" in scrape_item.url.parts:- async for soup in self.web_pager(videos_url):- check_soup(soup)- for new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):- new_scrape_item.append_folders("Videos")- self.create_task(self.run(new_scrape_item))+ async def gallery(self, scrape_item: ScrapeItem, gallery_id: str) -> None:+ for part in ("GI", "GV"):+ new_item = scrape_item.copy()+ new_item.url = self.PRIMARY_URL / f"{part}{gallery_id}"+ self.create_task(self.run(new_item))@error_handling_wrapper- async def collection(self, scrape_item: ScrapeItem) -> None:- group_id = scrape_item.url.name- gallery_id = group_id.removeprefix("G")- media_types = "images", "videos"- is_group = "g" in scrape_item.url.parts- if not is_group and gallery_id.startswith("G"): # No support for subgalleries- raise ScrapeError(422)+ async def group(self, scrape_item: ScrapeItem, slug: str) -> None:+ for part in ("gi", "gv"):+ new_item = scrape_item.copy()+ new_item.url = self.PRIMARY_URL / part / slug+ self.create_task(self.run(new_item))- for prefix in ("I", "V"):- gallery_id = gallery_id.removeprefix(prefix)-- if is_group:- new_urls = [PRIMARY_URL / part / group_id for part in ("gi", "gv")]+ @error_handling_wrapper+ async def user(self, scrape_item: ScrapeItem, username: str) -> None:+ for part in ("i", "v"):+ new_item = scrape_item.copy()+ new_item.url = (self.PRIMARY_URL / "u" / username).with_query(t=part)+ self.create_task(self.run(new_item))- else:- new_urls = [PRIMARY_URL / f"{part}{gallery_id}" for part in ("GI", "GV")]+ @error_handling_wrapper+ async def user_collection(self, scrape_item: ScrapeItem, username: str, name: str) -> None:+ scrape_item.setup_as_album(self.create_title(f"{username} [user]"))+ scrape_item.append_folders(name)- collection_id = group_id if is_group else gallery_id- for media_type, url in zip(media_types, new_urls, strict=True):- new_scrape_item = scrape_item.create_child(url)- await self.collection_items(new_scrape_item, media_type, collection_id)+ soup, pages = await aio.peek_first(self.web_pager(scrape_item.url))+ _check_soup(soup)+ await self._iter_media(scrape_item, pages)@error_handling_wrapper- async def collection_items(- self, scrape_item: ScrapeItem, media_type: Literal["videos", "images"], collection_id: str | None = None- ) -> None:+ async def collection(self, scrape_item: ScrapeItem, collection_id: str, name: str) -> None:soup, pages = await aio.peek_first(self.web_pager(scrape_item.url))- check_soup(soup)-- try:- title = css.select_text(soup, GALLERY_TITLE_SELECTOR)- except css.SelectorError:- title = css.select_text(soup, GROUP_TITLE_SELECTOR)-- title = self.create_title(title, collection_id)+ _check_soup(soup)+ title = self.create_title(css.select_text(soup, Selector.COLLECTION_TITLE), collection_id)scrape_item.setup_as_album(title, album_id=collection_id)+ scrape_item.append_folders(name)+ await self._iter_media(scrape_item, pages)+ async def _iter_media(self, scrape_item: ScrapeItem, pages: AsyncIterable[BeautifulSoup]) -> None:async for soup in pages:- for new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):- new_scrape_item.append_folders(media_type.capitalize())- self.create_task(self.run(new_scrape_item))+ for media in _extract_media_from_thumbs(soup):+ url = self.parse_url(media.href)+ new_item = scrape_item.create_child(url)+ if media.type == "image" and Path(url.name).stem == media.code:+ self.create_eager_task(self._media(new_item, media))+ else:+ self.create_eager_task(self.run(new_item))- @error_handling_wrapper- async def media(self, scrape_item: ScrapeItem) -> None:- media_id = scrape_item.url.parts[-1]- canonical_url = PRIMARY_URL / media_id+ scrape_item.add_children()- if await self.check_complete_from_referer(canonical_url):+ @error_handling_wrapper+ async def media(self, scrape_item: ScrapeItem, _media_id: str) -> None:+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)+ _check_soup(soup)+ await self._media(scrape_item, _extract_media(soup))- check_soup(soup)- media_info = self.process_media_soup(scrape_item, soup)- link = self.parse_url(media_info.url)- scrape_item.url = canonical_url- title = css.select_text(soup, ITEM_TITLE_SELECTOR)- filename, ext = self.get_filename_and_ext(link.name)- custom_filename = self.create_custom_filename(title, ext, file_id=media_id)- await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)-- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-- def process_media_soup(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> MediaInfo:- media_info = get_media_info(soup)- if media_info.type == "gallery":- raise ScrapeError(422)-- n_parts = len(scrape_item.url.parts)- from_gallery = n_parts > 2 and scrape_item.url.parts[1].startswith("G")- from_group = n_parts > 3 and "g" in scrape_item.url.parts- if not (from_gallery or from_group):- return media_info-- parent_id = scrape_item.url.parts[2] if from_group else scrape_item.url.parts[1]- parent_title = ""- title_tag = soup.select_one(ITEM_GALLERY_TITLE_SELECTOR)- if from_gallery:- parent_id = parent_id.removeprefix("G")- title_tag = soup.select_one(GROUP_TITLE_SELECTOR)-- if title_tag:- parent_title: str = css.attr(title_tag, "title") if from_gallery else title_tag.get_text(strip=True)-- parent_path = parent_id if from_gallery else f"g/{parent_id}"- parent_url = PRIMARY_URL / parent_path- if parent_url not in scrape_item.parents and parent_title:- scrape_item.parents.append(parent_url)- title = self.create_title(parent_title, parent_id)- scrape_item.setup_as_album(title, album_id=parent_id)- scrape_item.append_folders(f"{media_info.type.capitalize()}s")-- return media_info---def check_soup(soup: BeautifulSoup) -> None:- soup_str = soup.get_text()- if any(p in soup_str for p in NOT_FOUND_TEXTS):- raise ScrapeError(404)- if "The content you are trying to view is for friends only" in soup_str:+ @error_handling_wrapper+ async def _media(self, scrape_item: ScrapeItem, media: Media) -> None:+ scrape_item.upload_date = media.upload_date+ scrape_item.url = self.PRIMARY_URL / media.code+ _, ext = self.get_filename_and_ext(media.src.name)+ filename = self.create_custom_filename(media.name, ext, file_id=media.code)+ await self.handle_file(media.src, scrape_item, media.name, ext, custom_filename=filename)+++def _check_soup(soup: BeautifulSoup) -> None:+ html = soup.get_text()+ for txt in ("The page you're looking for cannot be found", "File not Found. Nothing to see here"):+ if txt in html:+ raise ScrapeError(404)+ if "The content you are trying to view is for friends only" in html:raise ScrapeError(401)-def get_media_info(soup: BeautifulSoup) -> MediaInfo:- media_js = soup.select_one(MEDIA_INFO_JS_SELECTOR)- js_text = css.text(media_js) if media_js else None- if not js_text:- return MediaInfo("gallery", "")- media_type = js_text.split("__mediatype", 1)[-1].split("=", 1)[-1].split(",", 1)[0].strip()- url = js_text.split("__fileurl", 1)[-1].split("=", 1)[-1].split(";", 1)[0].strip()- parts = remove_quotes(media_type.lower()), remove_quotes(url)- return MediaInfo(*parts)---def remove_quotes(text: str) -> str:- return text.removeprefix("'").removesuffix("'").removeprefix('"').removesuffix('"')[email protected](slots=True)+class Media:+ type: str+ code: str+ name: str+ href: str+ src: AbsoluteHttpURL+ upload_date: datetime.datetime | None+++def _extract_media_from_thumbs(soup: BeautifulSoup) -> Generator[Media]:+ for thumb in css.iselect(soup, Selector.THUMB):+ static = css.select(thumb, "img.static")+ media_type = css.attr(thumb, "data-mediatype")+ yield Media(+ type=media_type,+ code=css.attr(thumb, "data-codename"),+ name=css.attr(static, "alt"),+ href=css.select(thumb, "a.img-container", "href"),+ src=parse_url(css.attr(static, "src").replace("thumb", media_type)),+ upload_date=None,+ )+++def _extract_media(soup: BeautifulSoup) -> Media:+ js_text = css.select_text(soup, Selector.MEDIA_INFO_JS)+ extract = TextExtractor(js_text)+ props = css.json_ld(soup, "uploadDate")+ return Media(+ name=props["name"],+ upload_date=dates.parse_iso(props["uploadDate"]),+ type=extract("__mediatype = '", "'"),+ code=extract("__codename = '", "'"),+ href="",+ src=parse_url(extract("__fileurl = '", "'")),+ )diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.pyindex acf1effed..dcc6201eb 100644--- a/cyberdrop_dl/utils/__init__.py+++ b/cyberdrop_dl/utils/__init__.py@@ -29,11 +29,41 @@ def enter_context[T](context_var: ContextVar[T], value: T, /) -> Generator[None]context_var.reset(token)-def extr_text(text: str, /, start: str, end: str) -> str:- """Extracts the text between two strings in a larger text. Result will be stripped"""- start_index = text.index(start) + len(start)+def extract_text(text: str, /, start: str, end: str, pos: int | None = None) -> tuple[int, str]:+ """Extracts the text between two strings in a larger text.++ Result will be stripped"""+ start_index = text.index(start, pos) + len(start)end_index = text.index(end, start_index)- return text[start_index:end_index].strip()+ return end_index + len(end), text[start_index:end_index].strip()+++def extr_text(text: str, /, start: str, end: str) -> str:+ """Extracts the text between two strings in a larger text.++ Result will be stripped"""+ _, txt = extract_text(text, start, end)+ return txt+++class TextExtractor:+ def __init__(self, text: str, /, pos: int | None = None) -> None:+ self.text: str = text+ self.cursor: int | None = pos++ def __repr__(self) -> str:+ return f"{type(self).__name__}(text={self.text!r}, cursor={self.cursor!r})"++ def __call__(self, start: str, end: str) -> str:+ self.cursor, txt = extract_text(self.text, start, end, self.cursor)+ return txt++ def repeat(self, start: str, end: str) -> Generator[str]:+ while True:+ try:+ yield self(start, end)+ except ValueError:+ returndef get_system_information() -> dict[str, Any]:diff --git a/tests/crawlers/test_cases/motherless.py b/tests/crawlers/test_cases/motherless.pynew file mode 100644index 000000000..be821aecc--- /dev/null+++ b/tests/crawlers/test_cases/motherless.py@@ -0,0 +1,63 @@+DOMAIN = "motherless"+TEST_CASES = [+ {+ "url": "https://motherless.xxx/C03E162",+ "results": [+ {+ "url": "re:.motherlessmedia.com/images/C03E162.jpg",+ "filename": "C03E162 [C03E162].jpg",+ "debrid_url": None,+ "original_filename": "C03E162",+ "referer": "https://motherless.xxx/C03E162",+ "album_id": None,+ "uploaded_at": 1784409415,+ "download_folder": "re:Loose Files (Motherless)",+ }+ ],+ "count": 1,+ },+ {+ "url": "https://motherless.com/D1C90DF",+ "results": [+ {+ "url": "re:.motherlessmedia.com/videos/D1C90DF-720p.mp4",+ "filename": "Loot [D1C90DF].mp4",+ "debrid_url": None,+ "original_filename": "Loot",+ "referer": "https://motherless.xxx/D1C90DF",+ "album_id": None,+ "uploaded_at": 1784402707,+ "download_folder": "re:Loose Files (Motherless)",+ }+ ],+ "count": 1,+ },+ {+ "url": "https://motherless.xxx/G6323A2B/6D45EE7",+ "results": [+ {+ "url": "re:.motherlessmedia.com/videos/6D45EE7.mp4",+ "filename": "Ava [6D45EE7].mp4",+ "debrid_url": None,+ "original_filename": "Ava",+ "referer": "https://motherless.xxx/6D45EE7",+ "album_id": None,+ "uploaded_at": 1754164557,+ "download_folder": "re:Loose Files (Motherless)",+ }+ ],+ "count": 1,+ },+ {+ "url": "https://motherless.xxx/u/MysticX?t=v",+ "results": [+ {+ "url": "re:.motherlessmedia.com/videos/",+ "album_id": None,+ "uploaded_at": int,+ "download_folder": "re:MysticX [user] (Motherless)/videos",+ },+ ],+ "count": range(78, 90),+ },+]