Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add support for subfolders (dropbox) (#1286)
NTFSvolume34277bdb100b
diff --git a/cyberdrop_dl/crawlers/dropbox.py b/cyberdrop_dl/crawlers/dropbox.pyindex 683c5bba0..fe319f73a 100644--- a/cyberdrop_dl/crawlers/dropbox.py+++ b/cyberdrop_dl/crawlers/dropbox.py@@ -1,108 +1,178 @@from __future__ import annotations-from dataclasses import dataclass-from typing import TYPE_CHECKING, ClassVar+import dataclasses+from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.utils.utilities import error_handling_wrapper+from cyberdrop_dl.exceptions import LoginError, ScrapeError+from cyberdrop_dl.utils.utilities import error_handling_wrapper, type_adapterif TYPE_CHECKING:+ from collections.abc import AsyncGenerator+from cyberdrop_dl.data_structures.url_objects import ScrapeItem-ZIP_REFERENCE = (- "[--download-dropbox-folders-as-zip]"- "(https://script-ware.gitbook.io/cyberdrop-dl/reference/cli-arguments#download-dropbox-folders-as-zip)"-)-PRIMARY_URL = AbsoluteHttpURL("https://www.dropbox.com/")+_PRIMARY_URL = AbsoluteHttpURL("https://www.dropbox.com")+_FOLDERS_API_ENDPOINT = _PRIMARY_URL / "list_shared_link_folder_entries"++[email protected](slots=True)+class Node:+ is_dir: bool+ href: str+ filename: str+ secureHash: str = "" # noqa: N815+++_parse_node = type_adapter(Node)class DropboxCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"File": (- "/scl/fi/<file_id>","/s/...",- "/scl/fo/<token1>/<token2>?preview=<filename>",+ "/scl/fi/<link_key>?rlkey=...",+ "/scl/fo/<link_key>/<secure_hash>?preview=<filename>&rlkey=...",),"Folder": (- "/scl/fo/<token1>/<token2>","/sh/...",+ "/scl/fo/<link_key>/<secure_hash>?rlkey=...",),- "**NOTE**": f"Folders will be downloaded as a zip file. See: {ZIP_REFERENCE}",}- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URLDOMAIN: ClassVar[str] = "dropbox"- @property- def download_folders(self) -> bool:- return self.manager.parsed_args.cli_only_args.download_dropbox_folders_as_zip+ async def async_startup(self) -> None:+ await self._get_web_token(self.PRIMARY_URL)async def fetch(self, scrape_item: ScrapeItem) -> None:- """See https://www.dropboxforum.com/discussions/101001012/shared-link--scl-to-s/689070"""-- rlkey = scrape_item.url.query.get("rlkey") or ""- preview_filename = scrape_item.url.query.get("preview")match scrape_item.url.parts[1:]:case ["s" | "sh", _, *_]:- return await self.follow_redirect_w_head(scrape_item)- case ["scl", "fo", token1, token2]:- url = PRIMARY_URL / "scl/fo" / token1 / token2- if preview_filename:- url = url.with_query(preview=preview_filename)- folder_or_file = DropboxItem(None, rlkey, preview_filename, url)- return await self.folder_or_file(scrape_item, folder_or_file)- case ["scl", "fi", file_id, *file_name_part]:- url = PRIMARY_URL / "scl/fi" / file_id- if preview_filename:- file_name = preview_filename- elif file_name_part:- file_name = file_name_part[0]- else:- file_name = None- file = DropboxItem(file_id, rlkey, file_name, url)- return await self.folder_or_file(scrape_item, file)+ return await self.follow_redirect(scrape_item)+ case ["scl", "fi", _, *_]:+ return await self.file(scrape_item)+ case ["scl", "fo", link_key, secure_hash]:+ return await self.folder(scrape_item, link_key, secure_hash)+ case ["scl", "fo", link_key, secure_hash, _]:+ return await self.file(scrape_item)case _:raise ValueError@error_handling_wrapper- async def folder_or_file(self, scrape_item: ScrapeItem, item: DropboxItem) -> None:- if await self.check_complete_from_referer(item.view_url):- return-- if not item.rlkey:- raise ScrapeError(401)-- if item.is_folder and not self.download_folders:- raise ScrapeError(422, message="Folders download is not enabled")+ async def follow_redirect(self, scrape_item: ScrapeItem) -> None:+ async with self.request(scrape_item.url) as resp:+ if "s" in resp.url.parts or "sh" in resp.url.parts:+ if "error_pages/no_access" in await resp.text():+ raise ScrapeError(401)+ raise ScrapeError(422, "Infinite redirect")+ scrape_item.url = resp.url+ await self.fetch(scrape_item)- scrape_item.url = item.view_url- if item.filename:- filename = item.filename- else:- async with self.request(item.download_url) as resp:- filename = resp.filename-- filename, ext = self.get_filename_and_ext(filename)- await self.handle_file(item.url, scrape_item, filename, ext, debrid_link=item.download_url)---@dataclass(frozen=True, slots=True)-class DropboxItem:- id: str | None- rlkey: str- filename: str | None- url: AbsoluteHttpURL-- @property- def is_folder(self) -> bool:- return not bool(self.filename) and "fi" not in self.url.parts+ @error_handling_wrapper+ async def folder(self, scrape_item: ScrapeItem, link_key: str, secure_hash: str) -> None:+ scrape_item.url = await self._ensure_rlkey(scrape_item.url)+ rlkey = scrape_item.url.query["rlkey"]+ scrape_item.setup_as_album("")+ await self._walk_folder(scrape_item, link_key, secure_hash, rlkey)- @property- def download_url(self) -> AbsoluteHttpURL:- return self.url.update_query(dl=1, rlkey=self.rlkey)+ @error_handling_wrapper+ async def file(self, scrape_item: ScrapeItem) -> None:+ scrape_item.url = await self._ensure_rlkey(scrape_item.url)+ async with self.request(scrape_item.url.update_query(dl=1)) as resp:+ self._file(scrape_item, resp.filename)++ def _file(self, scrape_item: ScrapeItem, filename: str) -> None:+ scrape_item.url = view_url = scrape_item.url.with_query(rlkey=scrape_item.url.query["rlkey"], dl=0)+ download_url = view_url.update_query(dl=1)+ custom_filename, ext = self.get_filename_and_ext(filename)+ self.create_task(+ self.handle_file(+ view_url,+ scrape_item,+ filename,+ ext,+ debrid_link=download_url,+ custom_filename=custom_filename,+ )+ )++ async def _ensure_rlkey(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ if url.query.get("rlkey"):+ return url+ async with self.request(url) as resp:+ url = resp.url+ if url.query.get("rlkey"):+ return url+ raise ScrapeError(401)++ async def _walk_folder(+ self,+ scrape_item: ScrapeItem,+ link_key: str,+ secure_hash: str,+ rlkey: str,+ subpath: str = "",+ ) -> None:+ async for resp in self._web_api_pager(link_key, secure_hash, rlkey, subpath):+ if "folder" not in resp:+ return await self.file(scrape_item)+ folder_name: str = resp["folder"]["filename"]+ scrape_item.add_to_parent_title(self.create_title(folder_name))++ for entry, token in zip(resp["entries"], resp["share_tokens"], strict=True):+ node = _parse_node(token | entry)+ view_url = self.parse_url(node.href)+ new_scrape_item = scrape_item.create_child(view_url)+ if node.is_dir:+ self.create_task(+ self._walk_folder(+ new_scrape_item,+ link_key,+ node.secureHash,+ rlkey,+ f"{subpath}/{node.filename}",+ )+ )+ continue++ self._file(new_scrape_item, node.filename)+ scrape_item.add_children()++ async def _web_api_pager(+ self, link_key: str, secure_hash: str, rlkey: str, subpath: str = ""+ ) -> AsyncGenerator[dict[str, Any]]:+ payload = {+ "is_xhr": "true",+ "t": self._token,+ "sub_path": subpath,+ "link_key": link_key,+ "secure_hash": secure_hash,+ "rlkey": rlkey,+ "link_type": "s",+ }+ while True:+ resp: dict[str, Any] = await self.request_json(+ _FOLDERS_API_ENDPOINT,+ method="POST",+ data=payload,+ headers={+ "Origin": str(self.PRIMARY_URL),+ },+ )++ yield resp+ if not resp["has_more_entries"]:+ break+ payload["voucher"] = resp["next_request_voucher"]- @property- def view_url(self) -> AbsoluteHttpURL:- return self.url.with_query(rlkey=self.rlkey, e=1, dl=0)+ @error_handling_wrapper+ async def _get_web_token(self, *_):+ async with self.request(self.PRIMARY_URL, method="HEAD", cache_disabled=True):+ token = self.get_cookie_value("t")+ if not token:+ self.disabled = True+ msg = "Unable to get token from dropbox. Crawler has been disabled"+ raise LoginError(msg)+ self._token = tokendiff --git a/cyberdrop_dl/utils/args.py b/cyberdrop_dl/utils/args.pyindex 0c5acceb1..91d4b2603 100644--- a/cyberdrop_dl/utils/args.py+++ b/cyberdrop_dl/utils/args.py@@ -76,7 +76,6 @@ class CommandLineOnlyArgs(BaseModel):config_file: Path | None = Field(None, description="path to the CDL settings.yaml file to load")disable_cache: bool = Field(False, description="Temporarily disable the requests cache")download: bool = Field(False, description="skips UI, start download immediatly")- download_dropbox_folders_as_zip: bool = Field(False, description="download Dropbox folder without api key as zip")download_tiktok_audios: bool = Field(False, description="download TikTok audios")download_tiktok_src_quality_videos: bool = Field(False, description="download TikTok videos in source quality")max_items_retry: int = Field(0, description="max number of links to retry")diff --git a/tests/crawlers/test_cases/dropbox.py b/tests/crawlers/test_cases/dropbox.pynew file mode 100644index 000000000..29ec7f7d1--- /dev/null+++ b/tests/crawlers/test_cases/dropbox.py@@ -0,0 +1,48 @@+DOMAIN = "dropbox"+TEST_CASES = [+ (+ # folder with a single file inside+ "https://www.dropbox.com/scl/fo/6rtuuvcnhe8e0oivnfqtx/AO-LDPRIE_wqGhz8ZyLPWY0?rlkey=zbjcf274goirtfak2prmizi5p&dl=0",+ [+ {+ "url": "https://www.dropbox.com/scl/fo/6rtuuvcnhe8e0oivnfqtx/AAk0xkSvKleGaorbZfRJglA/Spherical-Cow-0.1.1.apk?rlkey=zbjcf274goirtfak2prmizi5p&dl=0",+ "debrid_url": "https://www.dropbox.com/scl/fo/6rtuuvcnhe8e0oivnfqtx/AAk0xkSvKleGaorbZfRJglA/Spherical-Cow-0.1.1.apk?rlkey=zbjcf274goirtfak2prmizi5p&dl=1",+ "filename": "Spherical-Cow-0.1.1.apk",+ "referer": "https://www.dropbox.com/scl/fo/6rtuuvcnhe8e0oivnfqtx/AAk0xkSvKleGaorbZfRJglA/Spherical-Cow-0.1.1.apk?rlkey=zbjcf274goirtfak2prmizi5p&dl=0",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ ("https://www.dropbox.com/scl/fo/igg7i2bazu3g689tqf0i2/h?rlkey=kwqk2vzmiebn9esfgxsm25kaz&dl=0", [], 15),+ (+ # folder with subfolder+ "https://www.dropbox.com/scl/fo/vdsh28mtrlz2dhgazoi6z/AFej9CAhhoEQJo4YDhlbZ0I?dl=0&rlkey=0887klgp6506j3jdy54dnwa6c",+ [+ {+ "url": "https://www.dropbox.com/scl/fo/vdsh28mtrlz2dhgazoi6z/ABWWQ94O950G1irCAniLZDw/Disk_1/sku.sis?rlkey=0887klgp6506j3jdy54dnwa6c&dl=0",+ "filename": "sku.sis",+ "download_folder": r"re:Hacknet Save backup \(Dropbox\)/Disk_1",+ },+ {+ "url": "https://www.dropbox.com/scl/fo/vdsh28mtrlz2dhgazoi6z/AJzG-YDVqQ5VGpGiWsainEM/Disk_1/365454_depotcache_1.csm?rlkey=0887klgp6506j3jdy54dnwa6c&dl=0",+ "filename": "365454_depotcache_1.csm",+ "download_folder": r"re:Hacknet Save backup \(Dropbox\)/Disk_1",+ },+ ],+ ),+ (+ # zipfile. It has a folder URL but is a single file+ "https://www.dropbox.com/scl/fo/vyuocyiqz1j93d71bdz18/ALdJdvzyF0lkH94c8dhztuw/afrojack_presskit_2014.zip?rlkey=hx60uhd3u1cecajczl1x9zn1l&e=1&dl=0",+ [+ {+ "url": "https://www.dropbox.com/scl/fo/vyuocyiqz1j93d71bdz18/ALdJdvzyF0lkH94c8dhztuw/afrojack_presskit_2014.zip?rlkey=hx60uhd3u1cecajczl1x9zn1l&dl=0",+ "debrid_url": "https://www.dropbox.com/scl/fo/vyuocyiqz1j93d71bdz18/ALdJdvzyF0lkH94c8dhztuw/afrojack_presskit_2014.zip?rlkey=hx60uhd3u1cecajczl1x9zn1l&dl=1",+ "filename": "afrojack_presskit_2014.zip",+ "referer": "https://www.dropbox.com/scl/fo/vyuocyiqz1j93d71bdz18/ALdJdvzyF0lkH94c8dhztuw/afrojack_presskit_2014.zip?rlkey=hx60uhd3u1cecajczl1x9zn1l&dl=0",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+]