Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: HTTP errors crashing CDL (MediaFire) (#1374)

NTFSvolume3aeae57bd0da
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 5f0d80f14..5b1a9b3bf 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -1,14 +1,17 @@
+"""https://www.mediafire.com/developers/core_api"""
+
from __future__ import annotations
import base64
+import dataclasses
import itertools
-from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple
+from typing import TYPE_CHECKING, Any, ClassVar, Literal
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import css
-from cyberdrop_dl.utils.utilities import error_handling_wrapper, is_blob_or_svg
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, is_blob_or_svg, type_adapter
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
@@ -18,19 +21,28 @@ if TYPE_CHECKING:
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-class Selector:
- DOWNLOAD_BUTTON = "a[id=downloadButton]"
- DATE = "ul[class=details] li:-soup-contains(Uploaded) span"
[email protected](slots=True, frozen=True)
+class Folder:
+ name: str
+ has_files: bool
+ has_folders: bool
+ @property
+ def is_empty(self) -> bool:
+ return not (self.has_files or self.has_folders)
-_PRIMARY_URL = AbsoluteHttpURL("https://www.mediafire.com/")
-_API_URL = _PRIMARY_URL / "api/1.4"
[email protected](slots=True, frozen=True)
+class File:
+ quickkey: str
+ filename: str
+ created: str
+ size: int
+ hash: str
-class FolderInfo(NamedTuple):
- name: str
- has_files: bool
- has_folders: bool
+
+_PRIMARY_URL = AbsoluteHttpURL("https://www.mediafire.com/")
+_API_URL = _PRIMARY_URL / "api/1.4"
class MediaFireCrawler(Crawler):
@@ -43,7 +55,10 @@ class MediaFireCrawler(Crawler):
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL
DOMAIN: ClassVar[str] = "mediafire"
- SKIP_PRE_CHECK = True
+ SKIP_PRE_CHECK: ClassVar[bool] = True
+
+ def __post_init__(self) -> None:
+ self.api = MediaFireAPI(self)
@classmethod
def _json_response_check(cls, json_resp: Any) -> None:
@@ -58,82 +73,42 @@ class MediaFireCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
if (
scrape_item.url.path == "/"
- and (file_id := scrape_item.url.query_string)
- and not ("&" in file_id or "=" in file_id)
+ and (quick_key := scrape_item.url.query_string)
+ and not ("&" in quick_key or "=" in quick_key)
):
- return await self.file(scrape_item, file_id)
+ return await self.file(scrape_item, quick_key)
match scrape_item.url.parts[1:]:
case ["folder", folder_key, *_]:
return await self.folder(scrape_item, folder_key)
- case ["file", file_id, *_]:
- return await self.file(scrape_item, file_id)
+ case ["file", quick_key, *_]:
+ return await self.file(scrape_item, quick_key)
case _:
raise ValueError
- async def _api_request(self, path: str, **params: Any) -> dict[str, Any]:
- params["response_format"] = "json"
- api_url = (_API_URL / path).with_query(params)
- return (await self.request_json(api_url))["response"]
-
- async def _iter_folder_content(self, folder_key: str, content_type: str) -> AsyncGenerator[list[dict[str, Any]]]:
- async def get_content(chunk: int) -> dict[str, Any]:
- return (
- await self._api_request(
- "folder/get_content.php",
- folder_key=folder_key,
- content_type=content_type,
- chunk=chunk,
- chunk_size=1000,
- filter="public",
- )
- )["folder_content"]
-
- next_task = self.create_task(get_content(1))
-
- for chunk in itertools.count(2):
- if next_task is None:
- break
- content = await next_task
- if content["more_chunks"] == "yes":
- next_task = self.create_task(get_content(chunk))
- else:
- next_task = None
- yield content[content_type]
-
- async def _get_folder_info(self, folder_key: str) -> FolderInfo:
- resp: dict[str, Any] = (
- await self._api_request(
- "folder/get_info.php",
- recursive="yes",
- details="yes",
- folder_key=folder_key,
- )
- )["folder_info"]
- return FolderInfo(
- name=resp["name"],
- has_files=bool(resp["file_count"]),
- has_folders=bool(resp["folder_count"]),
- )
-
@error_handling_wrapper
async def folder(self, scrape_item: ScrapeItem, folder_key: str) -> None:
- folder = await self._get_folder_info(folder_key)
+ folder = await self.api.folder_info(folder_key)
title = self.create_title(folder.name, folder_key)
scrape_item.setup_as_album(title, album_id=folder_key)
+ if folder.is_empty:
+ # Make a request anyway to try to get a descriptive error from MediaFire
+ async for _ in self.api.folder_content(folder_key, "folders"):
+ break
+ raise ScrapeError(204, "Folder is empty")
+
if folder.has_files:
- async for files in self._iter_folder_content(folder_key, "files"):
+ async for files in self.api.folder_content(folder_key, "files"):
for file in files:
- file_id: str = file["quickkey"]
- link = self.PRIMARY_URL / "file" / file_id
- new_scrape_item = scrape_item.create_child(link)
- new_scrape_item.possible_datetime = self.parse_iso_date(file["created"])
- self.create_task(self._file_task(new_scrape_item, file_id))
+ file_ = self.api.parse_file(file)
+ url = _PRIMARY_URL / "file" / file_.quickkey
+ new_scrape_item = scrape_item.create_child(url)
+ self.create_task(self._file_task(new_scrape_item, file_))
scrape_item.add_children()
if folder.has_folders:
- async for folders in self._iter_folder_content(folder_key, "folders"):
+ async for folders in self.api.folder_content(folder_key, "folders"):
for folder in folders:
link = self.PRIMARY_URL / "folder" / folder["folderkey"]
new_scrape_item = scrape_item.create_child(link)
@@ -141,36 +116,98 @@ class MediaFireCrawler(Crawler):
scrape_item.add_children()
@error_handling_wrapper
- async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:
- canonical_url = self.PRIMARY_URL / "file" / file_id
+ async def file(self, scrape_item: ScrapeItem, quick_key: str) -> None:
+ canonical_url = self.PRIMARY_URL / "file" / quick_key
if await self.check_complete_from_referer(canonical_url):
return
+ file = await self.api.file_info(quick_key)
+ scrape_item.url = canonical_url
+ await self._file(scrape_item, file, check_referer=False)
+
+ @error_handling_wrapper
+ async def _file(self, scrape_item: ScrapeItem, file: File, *, check_referer: bool = True) -> None:
+ if check_referer and await self.check_complete_from_referer(scrape_item):
+ return
+
+ hash_algo = "sha256" if len(file.hash) == 64 else "md5"
+ if await self.check_complete_by_hash(scrape_item, hash_algo, file.hash):
+ return
soup = await self.request_soup(scrape_item.url, impersonate=True)
- if not scrape_item.possible_datetime:
- scrape_item.possible_datetime = self.parse_iso_date(
- css.select_one_get_text(soup, Selector.DATE),
+ scrape_item.possible_datetime = self.parse_iso_date(file.created)
+ link = self.parse_url(_extract_download_link(soup))
+ filename, ext = self.get_filename_and_ext(file.filename)
+ await self.handle_file(link, scrape_item, file.filename, ext, custom_filename=filename)
+
+ _file_task = auto_task_id(_file)
+
+
+class MediaFireAPI:
+ def __init__(self, crawler: Crawler) -> None:
+ self._crawler = crawler
+ self.parse_file = type_adapter(File)
+
+ async def _api_request(self, path: str, **params: int | str) -> dict[str, Any]:
+ assert params
+ params["response_format"] = "json"
+ api_url = (_API_URL / path).with_query(params)
+ return (await self._crawler.request_json(api_url))["response"]
+
+ async def folder_content(
+ self, folder_key: str, content_type: Literal["files", "folders"]
+ ) -> AsyncGenerator[list[dict[str, Any]]]:
+ for chunk in itertools.count(1):
+ content = (
+ await self._api_request(
+ "folder/get_content.php",
+ folder_key=folder_key,
+ content_type=content_type,
+ chunk=chunk,
+ chunk_size=1000,
+ filter="public",
+ )
+ )["folder_content"]
+ yield content[content_type]
+ if content["more_chunks"] != "yes":
+ break
+
+ async def folder_info(self, folder_key: str) -> Folder:
+ resp: dict[str, Any] = (
+ await self._api_request(
+ "folder/get_info.php",
+ folder_key=folder_key,
)
+ )["folder_info"]
- scrape_item.url = canonical_url
- link = self.parse_url(_extract_download_link(soup))
- filename, ext = self.get_filename_and_ext(link.name)
- await self.handle_file(link, scrape_item, filename, ext)
+ return Folder(
+ name=resp["name"],
+ has_files=bool(int(resp["file_count"])),
+ has_folders=bool(
+ int(resp["folder_count"]),
+ ),
+ )
- _file_task = auto_task_id(file)
+ async def file_info(self, quick_key: str) -> File:
+ resp: dict[str, Any] = (
+ await self._api_request(
+ "file/get_info.php",
+ quick_key=quick_key,
+ )
+ )["file_info"]
+ return self.parse_file(resp)
def _extract_download_link(soup: BeautifulSoup) -> str:
- link_tag = soup.select_one(Selector.DOWNLOAD_BUTTON)
- if not link_tag:
+ download_button = soup.select_one("a#downloadButton")
+ if not download_button:
if "Something appears to be missing" in soup.get_text():
raise ScrapeError(410)
raise ScrapeError(422)
- if encoded_url := css.get_attr_or_none(link_tag, "data-scrambled-url"):
+ if encoded_url := css.get_attr_or_none(download_button, "data-scrambled-url"):
return base64.b64decode(encoded_url).decode()
- url = css.get_attr(link_tag, "href")
+ url = css.get_attr(download_button, "href")
if is_blob_or_svg(url):
raise ScrapeError(422)
return url
diff --git a/tests/crawlers/test_cases/mediafire.py b/tests/crawlers/test_cases/mediafire.py
index bf7b3c3d8..ba3d50df3 100644
--- a/tests/crawlers/test_cases/mediafire.py
+++ b/tests/crawlers/test_cases/mediafire.py
@@ -20,7 +20,7 @@ TEST_CASES = [
"filename": "ipwn1x-1.0.iso",
"referer": "https://www.mediafire.com/file/511jd1358yxvf26",
"album_id": None,
- "datetime": 1651019227,
+ "datetime": 1654132657,
}
],
),
@@ -31,9 +31,9 @@ TEST_CASES = [
"url": "re:https://download",
"filename": "kobo_clara.kepub.epub",
"referer": "https://www.mediafire.com/file/5iv342h2u39e0t6",
- "download_folder": r"re:kcc samples \(MediaFire\)",
+ "download_folder": r"re:kcc samples \(Mediafire\)",
"album_id": "ixh40veo6hrc5",
- "datetime": 1651019227,
+ "datetime": 1746643902,
},
],
11,
@@ -49,7 +49,7 @@ TEST_CASES = [
"album_id": "0m0g59yk1pmh5",
},
{
- "url": "re:https://download",
+ "url": "re:https://downloa",
"filename": "ReadMe.txt",
"referer": "https://www.mediafire.com/file/4tu93kyda5jp6qh",
"download_folder": r"re:XJTU-SY_Bearing_Datasets \(Mediafire\)",
@@ -70,4 +70,8 @@ TEST_CASES = [
[],
36,
),
+ (
+ "https://www.mediafire.com/folder/ujs6wzw0cecof/", # DMCA folder
+ [],
+ ),
]