Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: rework crawler (OneDrive) (#2122)
NTFSvolume3ac10102e670
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 93466ae79..0348e4352 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- `403 Forbidden` on all downloads (AdobeLightroom)- Parsing of embeded folder URLs (GoogleDrive)- `400 Bad request` for albums (PornHub)+- Single file downloads (OneDrive)- URL matching (Anysex)- Download of nested folders (Box.com)diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.pyindex a51b65833..f24d52252 100644--- a/cyberdrop_dl/crawlers/onedrive.py+++ b/cyberdrop_dl/crawlers/onedrive.py@@ -5,242 +5,223 @@from __future__ import annotationsimport dataclasses-import functools-from typing import TYPE_CHECKING, Any, ClassVar, Self+from collections import deque+from contextvars import ContextVar+from typing import TYPE_CHECKING, Any, ClassVar, Literalfrom cyberdrop_dl.cache import disk_cached_method-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, URLConfig+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedDomains, SupportedPaths, URLConfigfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import dates+from cyberdrop_dl.utils import dates, parse_url+from cyberdrop_dl.utils.dataclass import deserializefrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem--API_ENTRYPOINT = AbsoluteHttpURL("https://api.onedrive.com/v1.0/drives/")-PERSONAL_API_ENTRYPOINT = AbsoluteHttpURL("https://my.microsoftpersonalcontent.com/_api/v2.0/shares/")-BADGER_URL = AbsoluteHttpURL("https://api-badgerp.svc.ms/v1.0/token")-SHARE_LINK_HOST = "1drv.ms"-SHARE_LINK_PARTS = "f", "t", "u", "b"--# Default app details used in browsers by unautenticated sessions-APP_ID = "1141147648"-APP_UUID = "5cbed6ac-a083-4e14-b191-b4ba07653de2"--[email protected](frozen=True, slots=True)-class AccessDetails:- container_id: str- resid: str- auth_key: str- redeem: str-- @classmethod- def from_url(cls, direct_url: AbsoluteHttpURL) -> AccessDetails:- resid = direct_url.query.get("resid") or "" # ex: ABCXYZ000!12345- auth_key = direct_url.query.get("authkey") or ""- redeem = direct_url.query.get("redeem") or ""- id_ = direct_url.query.get("id") or ""- container_id = direct_url.query.get("cid")- if not resid and "!" in id_:- resid = id_- if not container_id:- container_id = resid.split("!")[0]-- return AccessDetails(container_id, resid, auth_key, redeem)--[email protected](frozen=True, slots=True)-class OneDriveItem:- id: str- url: AbsoluteHttpURL- web_url: AbsoluteHttpURL- name: str- date: int- access_details: AccessDetails--[email protected](frozen=True, slots=True)-class OneDriveFile(OneDriveItem):- download_url: AbsoluteHttpURL-- @classmethod- def from_api_response(cls, json_resp: dict[str, Any], access_details: AccessDetails) -> Self:- info = parse_api_response(json_resp, access_details)- download_url_str = json_resp["@content.downloadUrl"]- info["download_url"] = AbsoluteHttpURL(download_url_str, encoded="%" in download_url_str)- return cls(**info)--[email protected](frozen=True, slots=True)-class OneDriveFolder(OneDriveItem):- children: list[dict[str, Any]]-- @classmethod- def from_api_response(cls, json_resp: dict[str, Any], access_details: AccessDetails) -> Self:- info = parse_api_response(json_resp, access_details)- info["children"] = json_resp["children"]- return cls(**info)+SHARE_URL: ContextVar[AbsoluteHttpURL] = ContextVar("SHARE_URL")@URLConfig(allow_empty_path=True)[email protected]_path_builder("path_qs")class OneDriveCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Access Link": "https://onedrive.live.com/?authkey=<KEY>&id=<ID>&cid=<CID>",- "Share Link (anyone can access)": (- "https://1drv.ms/t/<KEY>",- "https://1drv.ms/f/<KEY>",- "https://1drv.ms/b/<KEY>",- "https://1drv.ms/u/<KEY>",- ),+ "Share Link (anyone can access)": ("https://1drv.ms/<path>",),}- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = SHARE_LINK_HOST, "onedrive.live.com"+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "1drv.ms", "onedrive.live.com"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://onedrive.com/")DOMAIN: ClassVar[str] = "onedrive"FOLDER_DOMAIN: ClassVar[str] = "OneDrive"def __post_init__(self) -> None:- self.auth_headers: dict[str, str] = {}+ self.api: OneDriveAPI = OneDriveAPI.from_crawler(self)async def __async_post_init__(self) -> None:- if self.auth_headers:- return- with self.catch_errors(BADGER_URL), self.disable_on_error("Unable to get badger token"):- token = await self.get_badger_token()- self.auth_headers = {"Prefer": "autoredeem", "Authorization": f"Badger {token}"}+ with self.disable_on_error("Unable to get badger token"):+ await self.api.auth()async def fetch(self, scrape_item: ScrapeItem) -> None:# ex: https://1drv.ms/t/s!ABCJKL-ABCJKL?e=ABC123 or https://1drv.ms/t/c/a12345678/aTOKEN?e=ABC123- if is_share_link(scrape_item.url):+ if scrape_item.url.host == "1drv.ms":return await self.share_link(scrape_item)# ex: https://onedrive.live.com/?authkey=!AUTHXXX-12345&id=ABCXYZ!12345&cid=ABC0123BVC- await self.link_with_credentials(scrape_item)+ await self.resource(scrape_item)@error_handling_wrapperasync def share_link(self, scrape_item: ScrapeItem) -> None:- og_share_link = scrape_item.url+ if await self.check_complete_from_referer(scrape_item.url):+ return+ SHARE_URL.set(scrape_item.url)scrape_item.url = await self.request_redirect(scrape_item.url)- await self.link_with_credentials(scrape_item, og_share_link)+ await self.resource(scrape_item)@error_handling_wrapper- async def link_with_credentials(- self, scrape_item: ScrapeItem, og_share_link: AbsoluteHttpURL | None = None- ) -> None:+ async def resource(self, scrape_item: ScrapeItem) -> None:+ resource_id, drive_id, cred = _parse_resource(scrape_item.url)if await self.check_complete_from_referer(scrape_item.url):return- if og_share_link and await self.check_complete_from_referer(og_share_link):+ resource = await self.api.resource(resource_id, drive_id, cred)+ if resource.type == "file":+ await self._file(scrape_item, resource, cred)return- access_details = AccessDetails.from_url(scrape_item.url)- await self.process_access_details(scrape_item, access_details, og_share_link)+ scrape_item.setup_as_album(self.create_title(resource.name))+ await self._walk_fs(scrape_item, resource, cred)- @error_handling_wrapper- async def process_access_details(- self, scrape_item: ScrapeItem, access_details: AccessDetails, og_share_link: AbsoluteHttpURL | None = None- ) -> None:- if not (access_details.resid and access_details.auth_key) and not access_details.redeem:- raise ScrapeError(401)+ async def _walk_fs(self, scrape_item: ScrapeItem, folder: Folder, cred: Credentials) -> None:+ subfolders: deque[tuple[tuple[str, ...], Folder]] = deque()+ path = ()- if access_details.redeem and not self.auth_headers:- raise ScrapeError(401)+ while True:+ children = await self.api.children(folder.id, folder.drive_id, cred)+ for node in children:+ if node.type == "folder":+ subfolders.append(((*path, node.name), node))+ continue- api_url = create_api_url(access_details)- json_resp: dict[str, Any] = await self.make_api_request(api_url)+ new_item = scrape_item.create_child(node.web_url)+ new_item.append_folders(*path)+ self.create_eager_task(self._file(new_item, node, cred))+ scrape_item.add_children()- if not is_folder(json_resp):- file = OneDriveFile.from_api_response(json_resp, access_details)- scrape_item.url = og_share_link or file.web_url- return await self.process_file(scrape_item, file)+ if not subfolders:+ return- folder = OneDriveFolder.from_api_response(json_resp, access_details)- await self.process_folder(scrape_item, folder)+ path, folder = subfolders.popleft()@error_handling_wrapper- async def process_folder(self, scrape_item: ScrapeItem, folder: OneDriveFolder) -> None:- title = self.create_title(folder.name)- scrape_item.setup_as_album(title)-- subfolders: list[AccessDetails] = []- old_ad = folder.access_details- new_access_details = functools.partial(AccessDetails, auth_key=old_ad.auth_key, redeem=old_ad.redeem)-- for item in folder.children:- if is_folder(item):- container_id: str = item["parentReference"]["driveId"]- resid: str = item["id"]- access_details = new_access_details(container_id, resid)- subfolders.append(access_details)- continue-- file = OneDriveFile.from_api_response(item, folder.access_details)- new_scrape_item = scrape_item.create_child(file.web_url)- await self.process_file(new_scrape_item, file)- scrape_item.add_children()-- for access_details in subfolders:- api_url = create_api_url(access_details)- new_scrape_item = scrape_item.create_child(api_url)- self.create_task(self.process_access_details(new_scrape_item, access_details))- scrape_item.add_children()-- @error_handling_wrapper- async def process_file(self, scrape_item: ScrapeItem, file: OneDriveFile) -> None:+ async def _file(self, scrape_item: ScrapeItem, file: File, cred: Credentials) -> None:# scrape_item.url should be web URL aka share link, ex: https://1drv.ms/t/s!ABCJKL-ABCJKL?e=ABC123# file.url should be API URL, ex: https://api.onedrive.com/v1.0/drives/<container_id>/items/<resid>?authkey=<auth_key># Auth key will be removed in database but a new one can be generated from scrape_item.url+ if file.sha256 and await self.check_complete_by_hash(scrape_item.url, "sha256", file.sha256):+ return+filename, ext = self.get_filename_and_ext(file.name)scrape_item.uploaded_at = file.date- await self.handle_file(file.url, scrape_item, filename, ext, debrid_link=file.download_url)+ await self.handle_file(+ self.api.resolve(file.id, file.drive_id, cred),+ scrape_item,+ file.name,+ ext,+ custom_filename=filename,+ debrid_link=file.download_url,+ referer=SHARE_URL.get(scrape_item.url),+ )++[email protected](frozen=True, slots=True)+class Credentials:+ auth_key: str+ redeem: str++[email protected](frozen=True, slots=True)+class Resource:+ id: str+ drive_id: str+ name: str+ date: float+ web_url: AbsoluteHttpURL- async def make_api_request(self, api_url: AbsoluteHttpURL) -> dict[str, Any]:- return await self.request_json(api_url, headers={"Content-Type": "application/json", **self.auth_headers})+[email protected](frozen=True, slots=True)+class File(Resource):+ type: Literal["file"]+ download_url: AbsoluteHttpURL+ sha256: str | None = None++[email protected](frozen=True, slots=True)+class Folder(Resource):+ type: Literal["folder"]+++class OneDriveAPI(API):+ # Default app details used in browsers by unautenticated sessions+ APP_ID: ClassVar[str] = "1141147648"+ APP_UUID: ClassVar[str] = "5cbed6ac-a083-4e14-b191-b4ba07653de2"++ BADGER_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api-badgerp.svc.ms/v1.0/token")+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api.onedrive.com/v1.0/drives/")+ PERSONAL_ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL(+ "https://my.microsoftpersonalcontent.com/_api/v2.0/shares/"+ )++ async def auth(self) -> None:+ token = await self._get_badger_token()+ self.__http_ctx__.headers.update({"Prefer": "autoredeem", "Authorization": f"Badger {token}"})@disk_cached_method("badger_token", ttl=3600 * 20)- async def get_badger_token(self) -> str:+ async def _get_badger_token(self) -> str:resp: dict[str, Any] = await self.request_json(- BADGER_URL,+ self.BADGER_URL,method="POST",- headers={"Content-Type": "application/json", "AppId": APP_ID},- json={"appId": APP_UUID},+ headers={"AppId": self.APP_ID},+ json={"appId": self.APP_UUID},)return resp["token"]--def is_share_link(url: AbsoluteHttpURL) -> bool:- return bool(url.host and url.host == SHARE_LINK_HOST) and any(p in url.parts for p in SHARE_LINK_PARTS)---def is_folder(json_resp: dict[str, Any]) -> bool:- return bool(json_resp.get("folder"))+ @classmethod+ def resolve(cls, resource_id: str, drive_id: str, cred: Credentials) -> AbsoluteHttpURL:+ if cred.redeem:+ api_url = cls.PERSONAL_ENTRYPOINT / f"u!{cred.redeem}" / "driveitem"+ else:+ api_url = cls.ENTRYPOINT / drive_id / "items" / resource_id++ if cred.auth_key:+ return api_url.update_query(authkey=cred.auth_key)+ return api_url++ async def resource(self, resource_id: str, drive_id: str, cred: Credentials) -> File | Folder:+ url = self.resolve(resource_id, drive_id, cred)+ resp = await self.request_json(url)+ return normalize_resource(resp)++ async def children(self, resource_id: str, drive_id: str, cred: Credentials) -> list[File | Folder]:+ url = self.resolve(resource_id, drive_id, cred)+ resp = await self.request_json(url.with_query(expand="children", orderby="folder,name"))+ return [normalize_resource(c) for c in resp.get("children", ())]+++def normalize_resource(resp: dict[str, Any]) -> File | Folder:+ dl_url = resp.get("@content.downloadUrl")+ is_folder = bool(resp.get("folder"))+ try:+ sha256 = resp["hashes"]["sha256Hash"]+ except LookupError:+ sha256 = None++ data = {+ "id": resp["id"],+ "drive_id": resp["parentReference"]["driveId"],+ "web_url": parse_url(resp["webUrl"]),+ "name": resp["name"],+ "type": "folder" if is_folder else "file",+ "date": dates.parse_iso(resp["fileSystemInfo"]["lastModifiedDateTime"]).timestamp(),+ "download_url": parse_url(dl_url) if dl_url else None,+ "sha256": sha256,+ }+ return deserialize(Folder if is_folder else File, data)-def parse_api_response(json_resp: dict[str, Any], access_details: AccessDetails) -> dict[str, Any]:- web_url_str: str = json_resp["webUrl"]- item_id = json_resp["id"]- date_str = json_resp["fileSystemInfo"]["lastModifiedDateTime"]- drive_id = json_resp["parentReference"]["driveId"]- new_access_details = AccessDetails(drive_id, item_id, access_details.auth_key, access_details.redeem)- return {- "id": item_id,- "url": create_api_url(new_access_details),- "web_url": AbsoluteHttpURL(web_url_str, encoded="%" in web_url_str),- "name": json_resp["name"],- "date": dates.parse_iso(date_str).timestamp(),- "access_details": new_access_details,- }+def _parse_resource(url: AbsoluteHttpURL) -> tuple[str, str, Credentials]:+ get = url.query.get+ cred = Credentials(get("authkey") or "", get("redeem") or "")+ if not (cred.auth_key or cred.redeem):+ raise ScrapeError.unsupported()+ resource_id = get("id") or ""+ resid = get("resid") or "" # ex: ABCXYZ000!12345+ if not resid and "!" in resource_id:+ resid = resource_id-def create_api_url(access_details: AccessDetails) -> AbsoluteHttpURL:- if access_details.redeem:- api_url = PERSONAL_API_ENTRYPOINT / f"u!{access_details.redeem}" / "driveitem"- else:- api_url = API_ENTRYPOINT / access_details.container_id / "items" / access_details.resid+ if not (resid and cred.auth_key) and not cred.redeem:+ raise ScrapeError(401)- api_url = api_url.with_query(expand="children", orderby="folder,name")- if access_details.auth_key:- return api_url.update_query(authkey=access_details.auth_key)- return api_url+ container_id = get("cid") or resid.partition("!")[0]+ return resid, container_id, creddiff --git a/tests/crawlers/test_cases/onedrive.py b/tests/crawlers/test_cases/onedrive.pynew file mode 100644index 000000000..d1298f395--- /dev/null+++ b/tests/crawlers/test_cases/onedrive.py@@ -0,0 +1,20 @@+DOMAIN = "onedrive"+TEST_CASES = [+ {+ "url": "https://1drv.ms/v/s!AtOM4nXMml6LhVtwL9LELS4JOEN7?e=294Cws",+ "results": [+ {+ "url": r"re:my\.microsoftpersonalcontent\.com/_api/v2.0/shares/(.*)/driveitem",+ "filename": "LG New York HDR UHD 4K Demo.ts",+ "debrid_url": "re:https://my.microsoftpersonalcontent.com/personal/8b5e9acc75e28cd3/_layouts/15/download.aspx?UniqueId=75e28cd3-9acc-205e-808b-db0200000000",+ "original_filename": "LG New York HDR UHD 4K Demo.ts",+ "referer": "https://1drv.ms/v/s!AtOM4nXMml6LhVtwL9LELS4JOEN7?e=294Cws",+ "album_id": None,+ "uploaded_at": 1678604476,+ "download_folder": "re:Loose Files (OneDrive)",+ "thumbnail": None,+ }+ ],+ "count": 1,+ }+]