Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: HTTP errors crashing CDL (MediaFire) (#1374)
NTFSvolume3aeae57bd0da
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 5f0d80f14..5b1a9b3bf 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -1,14 +1,17 @@+"""https://www.mediafire.com/developers/core_api"""+from __future__ import annotationsimport base64+import dataclassesimport itertools-from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple+from typing import TYPE_CHECKING, Any, ClassVar, Literalfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_idfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import css-from cyberdrop_dl.utils.utilities import error_handling_wrapper, is_blob_or_svg+from cyberdrop_dl.utils.utilities import error_handling_wrapper, is_blob_or_svg, type_adapterif TYPE_CHECKING:from collections.abc import AsyncGenerator@@ -18,19 +21,28 @@ if TYPE_CHECKING:from cyberdrop_dl.data_structures.url_objects import ScrapeItem-class Selector:- DOWNLOAD_BUTTON = "a[id=downloadButton]"- DATE = "ul[class=details] li:-soup-contains(Uploaded) span"[email protected](slots=True, frozen=True)+class Folder:+ name: str+ has_files: bool+ has_folders: bool+ @property+ def is_empty(self) -> bool:+ return not (self.has_files or self.has_folders)-_PRIMARY_URL = AbsoluteHttpURL("https://www.mediafire.com/")-_API_URL = _PRIMARY_URL / "api/1.4"[email protected](slots=True, frozen=True)+class File:+ quickkey: str+ filename: str+ created: str+ size: int+ hash: str-class FolderInfo(NamedTuple):- name: str- has_files: bool- has_folders: bool++_PRIMARY_URL = AbsoluteHttpURL("https://www.mediafire.com/")+_API_URL = _PRIMARY_URL / "api/1.4"class MediaFireCrawler(Crawler):@@ -43,7 +55,10 @@ class MediaFireCrawler(Crawler):}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URLDOMAIN: ClassVar[str] = "mediafire"- SKIP_PRE_CHECK = True+ SKIP_PRE_CHECK: ClassVar[bool] = True++ def __post_init__(self) -> None:+ self.api = MediaFireAPI(self)@classmethoddef _json_response_check(cls, json_resp: Any) -> None:@@ -58,82 +73,42 @@ class MediaFireCrawler(Crawler):async def fetch(self, scrape_item: ScrapeItem) -> None:if (scrape_item.url.path == "/"- and (file_id := scrape_item.url.query_string)- and not ("&" in file_id or "=" in file_id)+ and (quick_key := scrape_item.url.query_string)+ and not ("&" in quick_key or "=" in quick_key)):- return await self.file(scrape_item, file_id)+ return await self.file(scrape_item, quick_key)match scrape_item.url.parts[1:]:case ["folder", folder_key, *_]:return await self.folder(scrape_item, folder_key)- case ["file", file_id, *_]:- return await self.file(scrape_item, file_id)+ case ["file", quick_key, *_]:+ return await self.file(scrape_item, quick_key)case _:raise ValueError- async def _api_request(self, path: str, **params: Any) -> dict[str, Any]:- params["response_format"] = "json"- api_url = (_API_URL / path).with_query(params)- return (await self.request_json(api_url))["response"]-- async def _iter_folder_content(self, folder_key: str, content_type: str) -> AsyncGenerator[list[dict[str, Any]]]:- async def get_content(chunk: int) -> dict[str, Any]:- return (- await self._api_request(- "folder/get_content.php",- folder_key=folder_key,- content_type=content_type,- chunk=chunk,- chunk_size=1000,- filter="public",- )- )["folder_content"]-- next_task = self.create_task(get_content(1))-- for chunk in itertools.count(2):- if next_task is None:- break- content = await next_task- if content["more_chunks"] == "yes":- next_task = self.create_task(get_content(chunk))- else:- next_task = None- yield content[content_type]-- async def _get_folder_info(self, folder_key: str) -> FolderInfo:- resp: dict[str, Any] = (- await self._api_request(- "folder/get_info.php",- recursive="yes",- details="yes",- folder_key=folder_key,- )- )["folder_info"]- return FolderInfo(- name=resp["name"],- has_files=bool(resp["file_count"]),- has_folders=bool(resp["folder_count"]),- )-@error_handling_wrapperasync def folder(self, scrape_item: ScrapeItem, folder_key: str) -> None:- folder = await self._get_folder_info(folder_key)+ folder = await self.api.folder_info(folder_key)title = self.create_title(folder.name, folder_key)scrape_item.setup_as_album(title, album_id=folder_key)+ if folder.is_empty:+ # Make a request anyway to try to get a descriptive error from MediaFire+ async for _ in self.api.folder_content(folder_key, "folders"):+ break+ raise ScrapeError(204, "Folder is empty")+if folder.has_files:- async for files in self._iter_folder_content(folder_key, "files"):+ async for files in self.api.folder_content(folder_key, "files"):for file in files:- file_id: str = file["quickkey"]- link = self.PRIMARY_URL / "file" / file_id- new_scrape_item = scrape_item.create_child(link)- new_scrape_item.possible_datetime = self.parse_iso_date(file["created"])- self.create_task(self._file_task(new_scrape_item, file_id))+ file_ = self.api.parse_file(file)+ url = _PRIMARY_URL / "file" / file_.quickkey+ new_scrape_item = scrape_item.create_child(url)+ self.create_task(self._file_task(new_scrape_item, file_))scrape_item.add_children()if folder.has_folders:- async for folders in self._iter_folder_content(folder_key, "folders"):+ async for folders in self.api.folder_content(folder_key, "folders"):for folder in folders:link = self.PRIMARY_URL / "folder" / folder["folderkey"]new_scrape_item = scrape_item.create_child(link)@@ -141,36 +116,98 @@ class MediaFireCrawler(Crawler):scrape_item.add_children()@error_handling_wrapper- async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:- canonical_url = self.PRIMARY_URL / "file" / file_id+ async def file(self, scrape_item: ScrapeItem, quick_key: str) -> None:+ canonical_url = self.PRIMARY_URL / "file" / quick_keyif await self.check_complete_from_referer(canonical_url):return+ file = await self.api.file_info(quick_key)+ scrape_item.url = canonical_url+ await self._file(scrape_item, file, check_referer=False)++ @error_handling_wrapper+ async def _file(self, scrape_item: ScrapeItem, file: File, *, check_referer: bool = True) -> None:+ if check_referer and await self.check_complete_from_referer(scrape_item):+ return++ hash_algo = "sha256" if len(file.hash) == 64 else "md5"+ if await self.check_complete_by_hash(scrape_item, hash_algo, file.hash):+ returnsoup = await self.request_soup(scrape_item.url, impersonate=True)- if not scrape_item.possible_datetime:- scrape_item.possible_datetime = self.parse_iso_date(- css.select_one_get_text(soup, Selector.DATE),+ scrape_item.possible_datetime = self.parse_iso_date(file.created)+ link = self.parse_url(_extract_download_link(soup))+ filename, ext = self.get_filename_and_ext(file.filename)+ await self.handle_file(link, scrape_item, file.filename, ext, custom_filename=filename)++ _file_task = auto_task_id(_file)+++class MediaFireAPI:+ def __init__(self, crawler: Crawler) -> None:+ self._crawler = crawler+ self.parse_file = type_adapter(File)++ async def _api_request(self, path: str, **params: int | str) -> dict[str, Any]:+ assert params+ params["response_format"] = "json"+ api_url = (_API_URL / path).with_query(params)+ return (await self._crawler.request_json(api_url))["response"]++ async def folder_content(+ self, folder_key: str, content_type: Literal["files", "folders"]+ ) -> AsyncGenerator[list[dict[str, Any]]]:+ for chunk in itertools.count(1):+ content = (+ await self._api_request(+ "folder/get_content.php",+ folder_key=folder_key,+ content_type=content_type,+ chunk=chunk,+ chunk_size=1000,+ filter="public",+ )+ )["folder_content"]+ yield content[content_type]+ if content["more_chunks"] != "yes":+ break++ async def folder_info(self, folder_key: str) -> Folder:+ resp: dict[str, Any] = (+ await self._api_request(+ "folder/get_info.php",+ folder_key=folder_key,)+ )["folder_info"]- scrape_item.url = canonical_url- link = self.parse_url(_extract_download_link(soup))- filename, ext = self.get_filename_and_ext(link.name)- await self.handle_file(link, scrape_item, filename, ext)+ return Folder(+ name=resp["name"],+ has_files=bool(int(resp["file_count"])),+ has_folders=bool(+ int(resp["folder_count"]),+ ),+ )- _file_task = auto_task_id(file)+ async def file_info(self, quick_key: str) -> File:+ resp: dict[str, Any] = (+ await self._api_request(+ "file/get_info.php",+ quick_key=quick_key,+ )+ )["file_info"]+ return self.parse_file(resp)def _extract_download_link(soup: BeautifulSoup) -> str:- link_tag = soup.select_one(Selector.DOWNLOAD_BUTTON)- if not link_tag:+ download_button = soup.select_one("a#downloadButton")+ if not download_button:if "Something appears to be missing" in soup.get_text():raise ScrapeError(410)raise ScrapeError(422)- if encoded_url := css.get_attr_or_none(link_tag, "data-scrambled-url"):+ if encoded_url := css.get_attr_or_none(download_button, "data-scrambled-url"):return base64.b64decode(encoded_url).decode()- url = css.get_attr(link_tag, "href")+ url = css.get_attr(download_button, "href")if is_blob_or_svg(url):raise ScrapeError(422)return urldiff --git a/tests/crawlers/test_cases/mediafire.py b/tests/crawlers/test_cases/mediafire.pyindex bf7b3c3d8..ba3d50df3 100644--- a/tests/crawlers/test_cases/mediafire.py+++ b/tests/crawlers/test_cases/mediafire.py@@ -20,7 +20,7 @@ TEST_CASES = ["filename": "ipwn1x-1.0.iso","referer": "https://www.mediafire.com/file/511jd1358yxvf26","album_id": None,- "datetime": 1651019227,+ "datetime": 1654132657,}],),@@ -31,9 +31,9 @@ TEST_CASES = ["url": "re:https://download","filename": "kobo_clara.kepub.epub","referer": "https://www.mediafire.com/file/5iv342h2u39e0t6",- "download_folder": r"re:kcc samples \(MediaFire\)",+ "download_folder": r"re:kcc samples \(Mediafire\)","album_id": "ixh40veo6hrc5",- "datetime": 1651019227,+ "datetime": 1746643902,},],11,@@ -49,7 +49,7 @@ TEST_CASES = ["album_id": "0m0g59yk1pmh5",},{- "url": "re:https://download",+ "url": "re:https://downloa","filename": "ReadMe.txt","referer": "https://www.mediafire.com/file/4tu93kyda5jp6qh","download_folder": r"re:XJTU-SY_Bearing_Datasets \(Mediafire\)",@@ -70,4 +70,8 @@ TEST_CASES = [[],36,),+ (+ "https://www.mediafire.com/folder/ujs6wzw0cecof/", # DMCA folder+ [],+ ),]