Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: rework crawler (Box) (#2115)
NTFSvolume2bcf1d8b576a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 08e707256..9de7a9aea 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Support file paths as positional arguments- Support multiple files as input- Support mixing URLs and input files on the same run (all files must exists and all URLs/files must be positional arguments)+- Support sub-file/sub-folder URLs (Box.com)### Changed@@ -47,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Parsing of embeded folder URLs (GoogleDrive)- `400 Bad request` for albums (PornHub)- URL matching (Anysex)+- Download of nested folders (Box.com)## [10.7.0] - 2026-08-29diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.pyindex ec8f02af9..dd257baf5 100644--- a/cyberdrop_dl/crawlers/box_dot_com.py+++ b/cyberdrop_dl/crawlers/box_dot_com.py@@ -1,167 +1,256 @@from __future__ import annotations-import json-from collections import defaultdict-from enum import StrEnum-from pathlib import Path-from typing import TYPE_CHECKING, Any, ClassVar--from pydantic import Field--from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths-from cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.models import DeferredModel+import contextlib+import dataclasses+import itertools+from collections import deque+from pathlib import PurePosixPath+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self, TypedDict, override++from typing_extensions import AsyncGenerator, ReadOnly++from cyberdrop_dl import signature+from cyberdrop_dl.clients.http import HTTPConfig+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedDomains, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css+from cyberdrop_dl.utils.dataclass import deserializefrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:- from cyberdrop_dl.url_objects import ScrapeItem---class ItemType(StrEnum):- folder = "folder"- file = "file"---class Item(DeferredModel):- name: str- type: str- id: str = Field(validation_alias="itemID", coerce_numbers_to_str=True)- typed_id: str = Field("hola", validation_alias="typedID")- date: int | None = Field(default=None, validation_alias="contentUpdated")- parent_id: str = Field(validation_alias="parentFolderID", coerce_numbers_to_str=True)---class SharedFolder(DeferredModel):- name: str = Field(validation_alias="currentFolderName")- id: str = Field(validation_alias="currentFolderID", coerce_numbers_to_str=True)- items: list[Item]+ from collections.abc import Iterable+ from cyberdrop_dl.url_objects import ScrapeItem-APP_DOMAIN = "app.box.com"-DOWNLOAD_URL_BASE = AbsoluteHttpURL("https://app.box.com/index.php?rm=box_download_shared_file")-JS_SELECTOR = "script:-soup-contains('Box.postStreamData')"+APP_URL = AbsoluteHttpURL("https://app.box.com")+@HTTPConfig(headers={"Referer": str(APP_URL)})class BoxDotComCrawler(Crawler):- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (APP_DOMAIN,)+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (APP_URL.host,)SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "File or Folder": "app.box.com/s?sh=<share_code>",- "Embedded File or Folder": (- "app.box.com/embed/s?sh=<share_code>",- "app.box.com/embed_widget/s?sh=<share_code>",+ "Shared file/folder": (+ "/s?sh=<share_name>",+ "/s/<share_name>",+ "/s/<share_name>/file/<file_id>",+ "/s/<share_name>/folder/<folder_id>",+ "/embed/s?sh=<share_name>",+ "/embed_widget/s?sh=<share_name>",),}DOMAIN: ClassVar[str] = "box.com"FOLDER_DOMAIN: ClassVar[str] = "Box"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.box.com")+ def __post_init__(self) -> None:+ self.api: BoxDotComAPI = BoxDotComAPI.from_crawler(self)++ @classmethod+ @override+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ url = super().transform_url(url)+ match url.parts[1:]:+ case ["embed_widget" | "embed_widget", *_] if share_name := url.query.get("sh"):+ return APP_URL / "s" / share_name+ case ["shared" | "embed_widget" | "embed_widget", share_name]:+ return APP_URL / "s" / share_name+ case _:+ return url+async def fetch(self, scrape_item: ScrapeItem) -> None:- if scrape_item.url.host == APP_DOMAIN and ("s" in scrape_item.url.parts or scrape_item.url.query.get("s")):- return await self.file_or_folder(scrape_item)- raise ValueError+ match scrape_item.url.parts[1:]:+ case ["s", share_name, *rest]:+ match rest:+ case []:+ await self.share(scrape_item, share_name)+ case ["file", file_id]:+ await self.file(scrape_item, int(file_id), share_name)+ case ["folder", folder_id]:+ await self.folder(scrape_item, int(folder_id), share_name)+ case _:+ raise ValueError+ case ["s"] if share_name := scrape_item.url.query.get("sh"):+ await self.share(scrape_item, share_name)+ case _:+ raise ValueError@error_handling_wrapper- async def file_or_folder(self, scrape_item: ScrapeItem) -> None:- canonical_path = scrape_item.url.path- for trash in ("/embed_widget/", "/embed/"):- canonical_path = canonical_path.replace(trash, "")- scrape_item.url = scrape_item.url.with_path(canonical_path, keep_query=True, keep_fragment=True)- if "file" in scrape_item.url.parts and await self.check_complete_from_referer(scrape_item.url):- return None-- soup = await self.request_soup(scrape_item.url)-- if "file or folder link has been removed" in soup.get_text():- raise ScrapeError(410)-- js_text: str = css.select_text(soup, JS_SELECTOR)- data = js_text.removesuffix(";").partition("=")[-1]- if not data:- raise ScrapeError(422)-- info: dict[str, Any] = json.loads(data)- shared_name: str = info["/app-api/enduserapp/shared-item"]["sharedName"]- shared_folder_data: dict[str, Any] | None = info.get("/app-api/enduserapp/shared-folder")- if not shared_folder_data:- # This is a file direct URL, ex: https://app.box.com/s/f30ss109euq3r2yhsuics35acxmnm- # only the /file/ URL returns all the info about the file- # We need to re make the request- file_key = next(key for key in info if key.startswith("/app-api/enduserapp/item/f_"))- _, file_id = file_key.rsplit("f_", 1)- canonical_url = get_canonical_url(shared_name, file_id)- scrape_item.url = canonical_url- self.create_task(self.run(scrape_item))- return None-- shared_folder = SharedFolder(**shared_folder_data)- if "file" not in scrape_item.url.parts:- # Process folder- return await self.folder(scrape_item, shared_name, shared_folder)-- # Process individual file- assert len(scrape_item.url.parts) >= 5- file_id = scrape_item.url.parts[4]- file = next(item for item in shared_folder.items if item.id == file_id)- scrape_item.url = get_canonical_url(shared_name, file_id)- await self.file(scrape_item, shared_name, file)+ async def share(self, scrape_item: ScrapeItem, share_name: str) -> None:+ share = await self.api.share(share_name)+ if share.type == "file":+ await self.file(scrape_item, share.item_id, share.name)+ return++ await self.folder(scrape_item, share.item_id, share.name)@error_handling_wrapper- async def folder(self, scrape_item: ScrapeItem, shared_name: str, folder: SharedFolder) -> None:- title = self.create_title(folder.name, folder.id)- scrape_item.setup_as_album(title, album_id=folder.id)- scrape_item.url = get_canonical_url(shared_name, folder.id, is_folder=True)-- file_system = self.build_file_system(folder.items, folder.id)- for path, item in file_system.items():- if item.type != ItemType.file:- continue- link = get_canonical_url(shared_name, item.id)- new_scrape_item = scrape_item.create_child(link)- for part in path.parts:- new_scrape_item.append_folders(part)- await self.file(new_scrape_item, shared_name, item)- scrape_item.add_children()+ async def folder(self, scrape_item: ScrapeItem, folder_id: int, share_name: str):+ folder, get_nodes = await self.api.folder(folder_id, share_name)+ scrape_item.setup_as_album(self.create_title(folder.name, folder.share), album_id=folder.share)+ scrape_item.url = folder.url+ await self._walk_nodes(scrape_item, folder, get_nodes)@error_handling_wrapper- async def file(self, scrape_item: ScrapeItem, shared_name: str, file: Item) -> None:- assert file.type == ItemType.file+ async def file(self, scrape_item: ScrapeItem, file_id: int, share_name: str):+ file = await self.api.file(file_id, share_name)+ scrape_item.url = file.url+ await self._file(scrape_item, file)++ async def _walk_nodes(+ self,+ scrape_item: ScrapeItem,+ folder: Folder,+ get_nodes: AsyncGenerator[Iterable[Node]],+ ) -> None:+ subfolders: deque[int] = deque()+ seen: set[int] = set()++ while True:+ async with contextlib.aclosing(get_nodes) as pages:+ async for nodes in pages:+ for node in nodes:+ if node["id"] in seen:+ continue+ seen.add(node["id"])++ if node["type"] == "folder":+ subfolders.append(node["id"])+ continue++ if node["type"] != "file":+ self.log.warning("Unknown node type: %s", node)+ continue++ file = File.from_node(node, folder.share)+ new_item = scrape_item.create_child(file.url)+ new_item.append_folders(*folder.path.parts[1:])+ self.create_eager_task(self._file(new_item, file))+ scrape_item.add_children()++ if not subfolders:+ return++ folder, get_nodes = await self.api.folder(subfolders.popleft(), folder.share)++ @error_handling_wrapper+ async def _file(self, scrape_item: ScrapeItem, file: File) -> None:filename, ext = self.get_filename_and_ext(file.name)- link = DOWNLOAD_URL_BASE.update_query(shared_name=shared_name, file_id=file.typed_id)scrape_item.uploaded_at = file.date- await self.handle_file(scrape_item.url, scrape_item, filename, ext, debrid_link=link)-- def build_file_system(self, items: list[Item], root_id: str) -> dict[Path, Item]:- """Builds a flattened dictionary representing a file system from a list of items.+ await self.handle_file(+ scrape_item.url,+ scrape_item,+ file.name,+ ext,+ custom_filename=filename,+ debrid_link=self.api.download(file),+ )+++@HTTPConfig(headers={"X-Box-Client-Name": "enduserapp", "X-Box-Client-Version": "23.718.0", "Referer": str(APP_URL)})+class BoxDotComAPI(API):+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = APP_URL / "app-api/enduserapp"++ @signature.copy(API.request_json)+ async def request_json(self, *args, **kwargs) -> dict[str, Any]:+ async with self.request(*args, **kwargs) as resp:+ content = await resp.text()+ if content.lstrip().startswith("<!DOCTYPE html>"):+ resp.content_type = "text/html"+ return await resp.json()++ async def share(self, name: str) -> ShareItem:+ url = (self.ENTRYPOINT / "shared-item").with_query(sharedName=name)+ resp = await self.request_json(url)+ return ShareItem(item_id=resp["itemID"], name=resp["sharedName"], type=resp["itemType"])++ async def file(self, file_id: int, share_name: str) -> File:+ url = (self.ENTRYPOINT / f"item/f_{file_id}").with_query(format="preview")+ resp = await self.request_json(url, headers={"X-Box-EndUser-API": f"sharedName={share_name}"})+ return File.from_node(_normalize_node(resp["items"][0]), share_name)++ async def folder(self, folder_id: int, share_name: str) -> tuple[Folder, AsyncGenerator[map[Node]]]:+ url = (self.ENTRYPOINT / "shared-folder").with_query(folderID=folder_id)+ headers = {"X-Box-EndUser-API": f"sharedName={share_name}"}+ resp = await self.request_json(url, headers=headers)+ page_count: int = resp["pageCount"]++ async def nodes() -> AsyncGenerator[map[Node]]:+ nonlocal resp+ for page in itertools.count(1):+ yield map(_normalize_node, resp["items"])+ if page >= page_count:+ break+ resp = await self.request_json(url.update_query(page=page), headers=headers)++ return Folder.parse(resp["folder"], share_name), nodes()++ def download(self, file: File) -> AbsoluteHttpURL:+ return (APP_URL / "index.php").with_query(+ shared_name=file.share,+ file_id=f"f_{file.id}",+ rm="box_download_shared_file",+ )+++class Node(TypedDict):+ name: str+ type: ReadOnly[Literal["file", "folder"]]+ id: int+ typed_id: str+ date: int+ parent_id: int- Returns:- A 1-level dictionary where the each keys is the full path to a file/folder, and each value is the actual file/folder- """- path_mapping: dict[Path, Item] = {}- parents_mapping: dict[str, list[Item]] = defaultdict(list)[email protected](slots=True, frozen=True)+class ShareItem:+ item_id: int+ name: str+ type: Literal["file", "folder"]- for item in items:- parents_mapping[item.parent_id].append(item)- def build_tree(parent_id: str, current_path: Path) -> None:- for item in parents_mapping.get(parent_id, []):- item_path = current_path / item.name- path_mapping[item_path] = item[email protected](slots=True, frozen=True)+class File:+ id: int+ name: str+ date: int+ share: str- if item.type == ItemType.folder:- build_tree(item.id, item_path)+ @classmethod+ def from_node(cls, node: Node, share_name: str) -> Self:+ assert node["type"] == "file"+ return deserialize(cls, node, share=share_name)- root_item = next(item for item in items if item.id == root_id)- path = Path()- path_mapping[path] = root_item- build_tree(root_id, path)- return dict(sorted(path_mapping.items()))+ @property+ def url(self) -> AbsoluteHttpURL:+ return APP_URL / "s" / self.share / "file" / str(self.id)-def get_canonical_url(shared_name: str, share_id: str, *, is_folder: bool = False) -> AbsoluteHttpURL:- base = AbsoluteHttpURL(f"https://app.box.com/s/{shared_name}")- if is_folder:- return base / "folder" / share_id- return base / "file" / share_id[email protected](slots=True, frozen=True)+class Folder:+ id: int+ name: str+ share: str+ path: PurePosixPath++ @property+ def url(self) -> AbsoluteHttpURL:+ return APP_URL / "s" / self.share / f"folder/{self.id}"++ @classmethod+ def parse(cls, folder: dict[str, Any], share_name: str) -> Self:+ return cls(+ id=folder["id"],+ name=folder["name"],+ share=share_name,+ path=PurePosixPath(*(p["name"] for p in folder["path"])),+ )+++def _normalize_node(node: dict[str, Any]) -> Node:+ return {+ "name": node["name"],+ "type": node["type"],+ "id": node["id"],+ "typed_id": node["typedID"],+ "date": node.get("contentUpdated") or node["date"],+ "parent_id": node["parentFolderID"],+ }diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex e534cdff2..0e16a2684 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -892,7 +892,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):@final@contextlib.asynccontextmanager- async def new_task_group(self, scrape_item: ScrapeItem) -> AsyncGenerator[aio.EagerTaskGroup]:+ async def new_task_group(self, scrape_item: ScrapeItem | AbsoluteHttpURL) -> AsyncGenerator[aio.EagerTaskGroup]:async with aio.EagerTaskGroup() as tg:with self.catch_errors(scrape_item):yield tg