Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: rework crawler (OneDrive) (#2122)

NTFSvolume3ac10102e670
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 93466ae79..0348e4352 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `403 Forbidden` on all downloads (AdobeLightroom)
- Parsing of embeded folder URLs (GoogleDrive)
- `400 Bad request` for albums (PornHub)
+- Single file downloads (OneDrive)
- URL matching (Anysex)
- Download of nested folders (Box.com)
diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.py
index a51b65833..f24d52252 100644
--- a/cyberdrop_dl/crawlers/onedrive.py
+++ b/cyberdrop_dl/crawlers/onedrive.py
@@ -5,242 +5,223 @@
from __future__ import annotations
import dataclasses
-import functools
-from typing import TYPE_CHECKING, Any, ClassVar, Self
+from collections import deque
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, Any, ClassVar, Literal
from cyberdrop_dl.cache import disk_cached_method
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, URLConfig
+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedDomains, SupportedPaths, URLConfig
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import dates
+from cyberdrop_dl.utils import dates, parse_url
+from cyberdrop_dl.utils.dataclass import deserialize
from cyberdrop_dl.utils.errors import error_handling_wrapper
if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
-
-API_ENTRYPOINT = AbsoluteHttpURL("https://api.onedrive.com/v1.0/drives/")
-PERSONAL_API_ENTRYPOINT = AbsoluteHttpURL("https://my.microsoftpersonalcontent.com/_api/v2.0/shares/")
-BADGER_URL = AbsoluteHttpURL("https://api-badgerp.svc.ms/v1.0/token")
-SHARE_LINK_HOST = "1drv.ms"
-SHARE_LINK_PARTS = "f", "t", "u", "b"
-
-# Default app details used in browsers by unautenticated sessions
-APP_ID = "1141147648"
-APP_UUID = "5cbed6ac-a083-4e14-b191-b4ba07653de2"
-
-
[email protected](frozen=True, slots=True)
-class AccessDetails:
- container_id: str
- resid: str
- auth_key: str
- redeem: str
-
- @classmethod
- def from_url(cls, direct_url: AbsoluteHttpURL) -> AccessDetails:
- resid = direct_url.query.get("resid") or "" # ex: ABCXYZ000!12345
- auth_key = direct_url.query.get("authkey") or ""
- redeem = direct_url.query.get("redeem") or ""
- id_ = direct_url.query.get("id") or ""
- container_id = direct_url.query.get("cid")
- if not resid and "!" in id_:
- resid = id_
- if not container_id:
- container_id = resid.split("!")[0]
-
- return AccessDetails(container_id, resid, auth_key, redeem)
-
-
[email protected](frozen=True, slots=True)
-class OneDriveItem:
- id: str
- url: AbsoluteHttpURL
- web_url: AbsoluteHttpURL
- name: str
- date: int
- access_details: AccessDetails
-
-
[email protected](frozen=True, slots=True)
-class OneDriveFile(OneDriveItem):
- download_url: AbsoluteHttpURL
-
- @classmethod
- def from_api_response(cls, json_resp: dict[str, Any], access_details: AccessDetails) -> Self:
- info = parse_api_response(json_resp, access_details)
- download_url_str = json_resp["@content.downloadUrl"]
- info["download_url"] = AbsoluteHttpURL(download_url_str, encoded="%" in download_url_str)
- return cls(**info)
-
-
[email protected](frozen=True, slots=True)
-class OneDriveFolder(OneDriveItem):
- children: list[dict[str, Any]]
-
- @classmethod
- def from_api_response(cls, json_resp: dict[str, Any], access_details: AccessDetails) -> Self:
- info = parse_api_response(json_resp, access_details)
- info["children"] = json_resp["children"]
- return cls(**info)
+SHARE_URL: ContextVar[AbsoluteHttpURL] = ContextVar("SHARE_URL")
@URLConfig(allow_empty_path=True)
[email protected]_path_builder("path_qs")
class OneDriveCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Access Link": "https://onedrive.live.com/?authkey=<KEY>&id=<ID>&cid=<CID>",
- "Share Link (anyone can access)": (
- "https://1drv.ms/t/<KEY>",
- "https://1drv.ms/f/<KEY>",
- "https://1drv.ms/b/<KEY>",
- "https://1drv.ms/u/<KEY>",
- ),
+ "Share Link (anyone can access)": ("https://1drv.ms/<path>",),
}
- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = SHARE_LINK_HOST, "onedrive.live.com"
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "1drv.ms", "onedrive.live.com"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://onedrive.com/")
DOMAIN: ClassVar[str] = "onedrive"
FOLDER_DOMAIN: ClassVar[str] = "OneDrive"
def __post_init__(self) -> None:
- self.auth_headers: dict[str, str] = {}
+ self.api: OneDriveAPI = OneDriveAPI.from_crawler(self)
async def __async_post_init__(self) -> None:
- if self.auth_headers:
- return
- with self.catch_errors(BADGER_URL), self.disable_on_error("Unable to get badger token"):
- token = await self.get_badger_token()
- self.auth_headers = {"Prefer": "autoredeem", "Authorization": f"Badger {token}"}
+ with self.disable_on_error("Unable to get badger token"):
+ await self.api.auth()
async def fetch(self, scrape_item: ScrapeItem) -> None:
# ex: https://1drv.ms/t/s!ABCJKL-ABCJKL?e=ABC123 or https://1drv.ms/t/c/a12345678/aTOKEN?e=ABC123
- if is_share_link(scrape_item.url):
+ if scrape_item.url.host == "1drv.ms":
return await self.share_link(scrape_item)
# ex: https://onedrive.live.com/?authkey=!AUTHXXX-12345&id=ABCXYZ!12345&cid=ABC0123BVC
- await self.link_with_credentials(scrape_item)
+ await self.resource(scrape_item)
@error_handling_wrapper
async def share_link(self, scrape_item: ScrapeItem) -> None:
- og_share_link = scrape_item.url
+ if await self.check_complete_from_referer(scrape_item.url):
+ return
+ SHARE_URL.set(scrape_item.url)
scrape_item.url = await self.request_redirect(scrape_item.url)
- await self.link_with_credentials(scrape_item, og_share_link)
+ await self.resource(scrape_item)
@error_handling_wrapper
- async def link_with_credentials(
- self, scrape_item: ScrapeItem, og_share_link: AbsoluteHttpURL | None = None
- ) -> None:
+ async def resource(self, scrape_item: ScrapeItem) -> None:
+ resource_id, drive_id, cred = _parse_resource(scrape_item.url)
if await self.check_complete_from_referer(scrape_item.url):
return
- if og_share_link and await self.check_complete_from_referer(og_share_link):
+ resource = await self.api.resource(resource_id, drive_id, cred)
+ if resource.type == "file":
+ await self._file(scrape_item, resource, cred)
return
- access_details = AccessDetails.from_url(scrape_item.url)
- await self.process_access_details(scrape_item, access_details, og_share_link)
+ scrape_item.setup_as_album(self.create_title(resource.name))
+ await self._walk_fs(scrape_item, resource, cred)
- @error_handling_wrapper
- async def process_access_details(
- self, scrape_item: ScrapeItem, access_details: AccessDetails, og_share_link: AbsoluteHttpURL | None = None
- ) -> None:
- if not (access_details.resid and access_details.auth_key) and not access_details.redeem:
- raise ScrapeError(401)
+ async def _walk_fs(self, scrape_item: ScrapeItem, folder: Folder, cred: Credentials) -> None:
+ subfolders: deque[tuple[tuple[str, ...], Folder]] = deque()
+ path = ()
- if access_details.redeem and not self.auth_headers:
- raise ScrapeError(401)
+ while True:
+ children = await self.api.children(folder.id, folder.drive_id, cred)
+ for node in children:
+ if node.type == "folder":
+ subfolders.append(((*path, node.name), node))
+ continue
- api_url = create_api_url(access_details)
- json_resp: dict[str, Any] = await self.make_api_request(api_url)
+ new_item = scrape_item.create_child(node.web_url)
+ new_item.append_folders(*path)
+ self.create_eager_task(self._file(new_item, node, cred))
+ scrape_item.add_children()
- if not is_folder(json_resp):
- file = OneDriveFile.from_api_response(json_resp, access_details)
- scrape_item.url = og_share_link or file.web_url
- return await self.process_file(scrape_item, file)
+ if not subfolders:
+ return
- folder = OneDriveFolder.from_api_response(json_resp, access_details)
- await self.process_folder(scrape_item, folder)
+ path, folder = subfolders.popleft()
@error_handling_wrapper
- async def process_folder(self, scrape_item: ScrapeItem, folder: OneDriveFolder) -> None:
- title = self.create_title(folder.name)
- scrape_item.setup_as_album(title)
-
- subfolders: list[AccessDetails] = []
- old_ad = folder.access_details
- new_access_details = functools.partial(AccessDetails, auth_key=old_ad.auth_key, redeem=old_ad.redeem)
-
- for item in folder.children:
- if is_folder(item):
- container_id: str = item["parentReference"]["driveId"]
- resid: str = item["id"]
- access_details = new_access_details(container_id, resid)
- subfolders.append(access_details)
- continue
-
- file = OneDriveFile.from_api_response(item, folder.access_details)
- new_scrape_item = scrape_item.create_child(file.web_url)
- await self.process_file(new_scrape_item, file)
- scrape_item.add_children()
-
- for access_details in subfolders:
- api_url = create_api_url(access_details)
- new_scrape_item = scrape_item.create_child(api_url)
- self.create_task(self.process_access_details(new_scrape_item, access_details))
- scrape_item.add_children()
-
- @error_handling_wrapper
- async def process_file(self, scrape_item: ScrapeItem, file: OneDriveFile) -> None:
+ async def _file(self, scrape_item: ScrapeItem, file: File, cred: Credentials) -> None:
# scrape_item.url should be web URL aka share link, ex: https://1drv.ms/t/s!ABCJKL-ABCJKL?e=ABC123
# file.url should be API URL, ex: https://api.onedrive.com/v1.0/drives/<container_id>/items/<resid>?authkey=<auth_key>
# Auth key will be removed in database but a new one can be generated from scrape_item.url
+ if file.sha256 and await self.check_complete_by_hash(scrape_item.url, "sha256", file.sha256):
+ return
+
filename, ext = self.get_filename_and_ext(file.name)
scrape_item.uploaded_at = file.date
- await self.handle_file(file.url, scrape_item, filename, ext, debrid_link=file.download_url)
+ await self.handle_file(
+ self.api.resolve(file.id, file.drive_id, cred),
+ scrape_item,
+ file.name,
+ ext,
+ custom_filename=filename,
+ debrid_link=file.download_url,
+ referer=SHARE_URL.get(scrape_item.url),
+ )
+
+
[email protected](frozen=True, slots=True)
+class Credentials:
+ auth_key: str
+ redeem: str
+
+
[email protected](frozen=True, slots=True)
+class Resource:
+ id: str
+ drive_id: str
+ name: str
+ date: float
+ web_url: AbsoluteHttpURL
- async def make_api_request(self, api_url: AbsoluteHttpURL) -> dict[str, Any]:
- return await self.request_json(api_url, headers={"Content-Type": "application/json", **self.auth_headers})
+
[email protected](frozen=True, slots=True)
+class File(Resource):
+ type: Literal["file"]
+ download_url: AbsoluteHttpURL
+ sha256: str | None = None
+
+
[email protected](frozen=True, slots=True)
+class Folder(Resource):
+ type: Literal["folder"]
+
+
+class OneDriveAPI(API):
+ # Default app details used in browsers by unautenticated sessions
+ APP_ID: ClassVar[str] = "1141147648"
+ APP_UUID: ClassVar[str] = "5cbed6ac-a083-4e14-b191-b4ba07653de2"
+
+ BADGER_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api-badgerp.svc.ms/v1.0/token")
+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api.onedrive.com/v1.0/drives/")
+ PERSONAL_ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL(
+ "https://my.microsoftpersonalcontent.com/_api/v2.0/shares/"
+ )
+
+ async def auth(self) -> None:
+ token = await self._get_badger_token()
+ self.__http_ctx__.headers.update({"Prefer": "autoredeem", "Authorization": f"Badger {token}"})
@disk_cached_method("badger_token", ttl=3600 * 20)
- async def get_badger_token(self) -> str:
+ async def _get_badger_token(self) -> str:
resp: dict[str, Any] = await self.request_json(
- BADGER_URL,
+ self.BADGER_URL,
method="POST",
- headers={"Content-Type": "application/json", "AppId": APP_ID},
- json={"appId": APP_UUID},
+ headers={"AppId": self.APP_ID},
+ json={"appId": self.APP_UUID},
)
return resp["token"]
-
-def is_share_link(url: AbsoluteHttpURL) -> bool:
- return bool(url.host and url.host == SHARE_LINK_HOST) and any(p in url.parts for p in SHARE_LINK_PARTS)
-
-
-def is_folder(json_resp: dict[str, Any]) -> bool:
- return bool(json_resp.get("folder"))
+ @classmethod
+ def resolve(cls, resource_id: str, drive_id: str, cred: Credentials) -> AbsoluteHttpURL:
+ if cred.redeem:
+ api_url = cls.PERSONAL_ENTRYPOINT / f"u!{cred.redeem}" / "driveitem"
+ else:
+ api_url = cls.ENTRYPOINT / drive_id / "items" / resource_id
+
+ if cred.auth_key:
+ return api_url.update_query(authkey=cred.auth_key)
+ return api_url
+
+ async def resource(self, resource_id: str, drive_id: str, cred: Credentials) -> File | Folder:
+ url = self.resolve(resource_id, drive_id, cred)
+ resp = await self.request_json(url)
+ return normalize_resource(resp)
+
+ async def children(self, resource_id: str, drive_id: str, cred: Credentials) -> list[File | Folder]:
+ url = self.resolve(resource_id, drive_id, cred)
+ resp = await self.request_json(url.with_query(expand="children", orderby="folder,name"))
+ return [normalize_resource(c) for c in resp.get("children", ())]
+
+
+def normalize_resource(resp: dict[str, Any]) -> File | Folder:
+ dl_url = resp.get("@content.downloadUrl")
+ is_folder = bool(resp.get("folder"))
+ try:
+ sha256 = resp["hashes"]["sha256Hash"]
+ except LookupError:
+ sha256 = None
+
+ data = {
+ "id": resp["id"],
+ "drive_id": resp["parentReference"]["driveId"],
+ "web_url": parse_url(resp["webUrl"]),
+ "name": resp["name"],
+ "type": "folder" if is_folder else "file",
+ "date": dates.parse_iso(resp["fileSystemInfo"]["lastModifiedDateTime"]).timestamp(),
+ "download_url": parse_url(dl_url) if dl_url else None,
+ "sha256": sha256,
+ }
+ return deserialize(Folder if is_folder else File, data)
-def parse_api_response(json_resp: dict[str, Any], access_details: AccessDetails) -> dict[str, Any]:
- web_url_str: str = json_resp["webUrl"]
- item_id = json_resp["id"]
- date_str = json_resp["fileSystemInfo"]["lastModifiedDateTime"]
- drive_id = json_resp["parentReference"]["driveId"]
- new_access_details = AccessDetails(drive_id, item_id, access_details.auth_key, access_details.redeem)
- return {
- "id": item_id,
- "url": create_api_url(new_access_details),
- "web_url": AbsoluteHttpURL(web_url_str, encoded="%" in web_url_str),
- "name": json_resp["name"],
- "date": dates.parse_iso(date_str).timestamp(),
- "access_details": new_access_details,
- }
+def _parse_resource(url: AbsoluteHttpURL) -> tuple[str, str, Credentials]:
+ get = url.query.get
+ cred = Credentials(get("authkey") or "", get("redeem") or "")
+ if not (cred.auth_key or cred.redeem):
+ raise ScrapeError.unsupported()
+ resource_id = get("id") or ""
+ resid = get("resid") or "" # ex: ABCXYZ000!12345
+ if not resid and "!" in resource_id:
+ resid = resource_id
-def create_api_url(access_details: AccessDetails) -> AbsoluteHttpURL:
- if access_details.redeem:
- api_url = PERSONAL_API_ENTRYPOINT / f"u!{access_details.redeem}" / "driveitem"
- else:
- api_url = API_ENTRYPOINT / access_details.container_id / "items" / access_details.resid
+ if not (resid and cred.auth_key) and not cred.redeem:
+ raise ScrapeError(401)
- api_url = api_url.with_query(expand="children", orderby="folder,name")
- if access_details.auth_key:
- return api_url.update_query(authkey=access_details.auth_key)
- return api_url
+ container_id = get("cid") or resid.partition("!")[0]
+ return resid, container_id, cred
diff --git a/tests/crawlers/test_cases/onedrive.py b/tests/crawlers/test_cases/onedrive.py
new file mode 100644
index 000000000..d1298f395
--- /dev/null
+++ b/tests/crawlers/test_cases/onedrive.py
@@ -0,0 +1,20 @@
+DOMAIN = "onedrive"
+TEST_CASES = [
+ {
+ "url": "https://1drv.ms/v/s!AtOM4nXMml6LhVtwL9LELS4JOEN7?e=294Cws",
+ "results": [
+ {
+ "url": r"re:my\.microsoftpersonalcontent\.com/_api/v2.0/shares/(.*)/driveitem",
+ "filename": "LG New York HDR UHD 4K Demo.ts",
+ "debrid_url": "re:https://my.microsoftpersonalcontent.com/personal/8b5e9acc75e28cd3/_layouts/15/download.aspx?UniqueId=75e28cd3-9acc-205e-808b-db0200000000",
+ "original_filename": "LG New York HDR UHD 4K Demo.ts",
+ "referer": "https://1drv.ms/v/s!AtOM4nXMml6LhVtwL9LELS4JOEN7?e=294Cws",
+ "album_id": None,
+ "uploaded_at": 1678604476,
+ "download_folder": "re:Loose Files (OneDrive)",
+ "thumbnail": None,
+ }
+ ],
+ "count": 1,
+ }
+]