Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: rework crawler (Box) (#2115)

NTFSvolume2bcf1d8b576a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08e707256..9de7a9aea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Support file paths as positional arguments
- Support multiple files as input
- Support mixing URLs and input files on the same run (all files must exists and all URLs/files must be positional arguments)
+- Support sub-file/sub-folder URLs (Box.com)
### Changed
@@ -47,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Parsing of embeded folder URLs (GoogleDrive)
- `400 Bad request` for albums (PornHub)
- URL matching (Anysex)
+- Download of nested folders (Box.com)
## [10.7.0] - 2026-08-29
diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.py
index ec8f02af9..dd257baf5 100644
--- a/cyberdrop_dl/crawlers/box_dot_com.py
+++ b/cyberdrop_dl/crawlers/box_dot_com.py
@@ -1,167 +1,256 @@
from __future__ import annotations
-import json
-from collections import defaultdict
-from enum import StrEnum
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, ClassVar
-
-from pydantic import Field
-
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
-from cyberdrop_dl.exceptions import ScrapeError
-from cyberdrop_dl.models import DeferredModel
+import contextlib
+import dataclasses
+import itertools
+from collections import deque
+from pathlib import PurePosixPath
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self, TypedDict, override
+
+from typing_extensions import AsyncGenerator, ReadOnly
+
+from cyberdrop_dl import signature
+from cyberdrop_dl.clients.http import HTTPConfig
+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedDomains, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css
+from cyberdrop_dl.utils.dataclass import deserialize
from cyberdrop_dl.utils.errors import error_handling_wrapper
if TYPE_CHECKING:
- from cyberdrop_dl.url_objects import ScrapeItem
-
-
-class ItemType(StrEnum):
- folder = "folder"
- file = "file"
-
-
-class Item(DeferredModel):
- name: str
- type: str
- id: str = Field(validation_alias="itemID", coerce_numbers_to_str=True)
- typed_id: str = Field("hola", validation_alias="typedID")
- date: int | None = Field(default=None, validation_alias="contentUpdated")
- parent_id: str = Field(validation_alias="parentFolderID", coerce_numbers_to_str=True)
-
-
-class SharedFolder(DeferredModel):
- name: str = Field(validation_alias="currentFolderName")
- id: str = Field(validation_alias="currentFolderID", coerce_numbers_to_str=True)
- items: list[Item]
+ from collections.abc import Iterable
+ from cyberdrop_dl.url_objects import ScrapeItem
-APP_DOMAIN = "app.box.com"
-DOWNLOAD_URL_BASE = AbsoluteHttpURL("https://app.box.com/index.php?rm=box_download_shared_file")
-JS_SELECTOR = "script:-soup-contains('Box.postStreamData')"
+APP_URL = AbsoluteHttpURL("https://app.box.com")
+@HTTPConfig(headers={"Referer": str(APP_URL)})
class BoxDotComCrawler(Crawler):
- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (APP_DOMAIN,)
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (APP_URL.host,)
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "File or Folder": "app.box.com/s?sh=<share_code>",
- "Embedded File or Folder": (
- "app.box.com/embed/s?sh=<share_code>",
- "app.box.com/embed_widget/s?sh=<share_code>",
+ "Shared file/folder": (
+ "/s?sh=<share_name>",
+ "/s/<share_name>",
+ "/s/<share_name>/file/<file_id>",
+ "/s/<share_name>/folder/<folder_id>",
+ "/embed/s?sh=<share_name>",
+ "/embed_widget/s?sh=<share_name>",
),
}
DOMAIN: ClassVar[str] = "box.com"
FOLDER_DOMAIN: ClassVar[str] = "Box"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.box.com")
+ def __post_init__(self) -> None:
+ self.api: BoxDotComAPI = BoxDotComAPI.from_crawler(self)
+
+ @classmethod
+ @override
+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ url = super().transform_url(url)
+ match url.parts[1:]:
+ case ["embed_widget" | "embed_widget", *_] if share_name := url.query.get("sh"):
+ return APP_URL / "s" / share_name
+ case ["shared" | "embed_widget" | "embed_widget", share_name]:
+ return APP_URL / "s" / share_name
+ case _:
+ return url
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if scrape_item.url.host == APP_DOMAIN and ("s" in scrape_item.url.parts or scrape_item.url.query.get("s")):
- return await self.file_or_folder(scrape_item)
- raise ValueError
+ match scrape_item.url.parts[1:]:
+ case ["s", share_name, *rest]:
+ match rest:
+ case []:
+ await self.share(scrape_item, share_name)
+ case ["file", file_id]:
+ await self.file(scrape_item, int(file_id), share_name)
+ case ["folder", folder_id]:
+ await self.folder(scrape_item, int(folder_id), share_name)
+ case _:
+ raise ValueError
+ case ["s"] if share_name := scrape_item.url.query.get("sh"):
+ await self.share(scrape_item, share_name)
+ case _:
+ raise ValueError
@error_handling_wrapper
- async def file_or_folder(self, scrape_item: ScrapeItem) -> None:
- canonical_path = scrape_item.url.path
- for trash in ("/embed_widget/", "/embed/"):
- canonical_path = canonical_path.replace(trash, "")
- scrape_item.url = scrape_item.url.with_path(canonical_path, keep_query=True, keep_fragment=True)
- if "file" in scrape_item.url.parts and await self.check_complete_from_referer(scrape_item.url):
- return None
-
- soup = await self.request_soup(scrape_item.url)
-
- if "file or folder link has been removed" in soup.get_text():
- raise ScrapeError(410)
-
- js_text: str = css.select_text(soup, JS_SELECTOR)
- data = js_text.removesuffix(";").partition("=")[-1]
- if not data:
- raise ScrapeError(422)
-
- info: dict[str, Any] = json.loads(data)
- shared_name: str = info["/app-api/enduserapp/shared-item"]["sharedName"]
- shared_folder_data: dict[str, Any] | None = info.get("/app-api/enduserapp/shared-folder")
- if not shared_folder_data:
- # This is a file direct URL, ex: https://app.box.com/s/f30ss109euq3r2yhsuics35acxmnm
- # only the /file/ URL returns all the info about the file
- # We need to re make the request
- file_key = next(key for key in info if key.startswith("/app-api/enduserapp/item/f_"))
- _, file_id = file_key.rsplit("f_", 1)
- canonical_url = get_canonical_url(shared_name, file_id)
- scrape_item.url = canonical_url
- self.create_task(self.run(scrape_item))
- return None
-
- shared_folder = SharedFolder(**shared_folder_data)
- if "file" not in scrape_item.url.parts:
- # Process folder
- return await self.folder(scrape_item, shared_name, shared_folder)
-
- # Process individual file
- assert len(scrape_item.url.parts) >= 5
- file_id = scrape_item.url.parts[4]
- file = next(item for item in shared_folder.items if item.id == file_id)
- scrape_item.url = get_canonical_url(shared_name, file_id)
- await self.file(scrape_item, shared_name, file)
+ async def share(self, scrape_item: ScrapeItem, share_name: str) -> None:
+ share = await self.api.share(share_name)
+ if share.type == "file":
+ await self.file(scrape_item, share.item_id, share.name)
+ return
+
+ await self.folder(scrape_item, share.item_id, share.name)
@error_handling_wrapper
- async def folder(self, scrape_item: ScrapeItem, shared_name: str, folder: SharedFolder) -> None:
- title = self.create_title(folder.name, folder.id)
- scrape_item.setup_as_album(title, album_id=folder.id)
- scrape_item.url = get_canonical_url(shared_name, folder.id, is_folder=True)
-
- file_system = self.build_file_system(folder.items, folder.id)
- for path, item in file_system.items():
- if item.type != ItemType.file:
- continue
- link = get_canonical_url(shared_name, item.id)
- new_scrape_item = scrape_item.create_child(link)
- for part in path.parts:
- new_scrape_item.append_folders(part)
- await self.file(new_scrape_item, shared_name, item)
- scrape_item.add_children()
+ async def folder(self, scrape_item: ScrapeItem, folder_id: int, share_name: str):
+ folder, get_nodes = await self.api.folder(folder_id, share_name)
+ scrape_item.setup_as_album(self.create_title(folder.name, folder.share), album_id=folder.share)
+ scrape_item.url = folder.url
+ await self._walk_nodes(scrape_item, folder, get_nodes)
@error_handling_wrapper
- async def file(self, scrape_item: ScrapeItem, shared_name: str, file: Item) -> None:
- assert file.type == ItemType.file
+ async def file(self, scrape_item: ScrapeItem, file_id: int, share_name: str):
+ file = await self.api.file(file_id, share_name)
+ scrape_item.url = file.url
+ await self._file(scrape_item, file)
+
+ async def _walk_nodes(
+ self,
+ scrape_item: ScrapeItem,
+ folder: Folder,
+ get_nodes: AsyncGenerator[Iterable[Node]],
+ ) -> None:
+ subfolders: deque[int] = deque()
+ seen: set[int] = set()
+
+ while True:
+ async with contextlib.aclosing(get_nodes) as pages:
+ async for nodes in pages:
+ for node in nodes:
+ if node["id"] in seen:
+ continue
+ seen.add(node["id"])
+
+ if node["type"] == "folder":
+ subfolders.append(node["id"])
+ continue
+
+ if node["type"] != "file":
+ self.log.warning("Unknown node type: %s", node)
+ continue
+
+ file = File.from_node(node, folder.share)
+ new_item = scrape_item.create_child(file.url)
+ new_item.append_folders(*folder.path.parts[1:])
+ self.create_eager_task(self._file(new_item, file))
+ scrape_item.add_children()
+
+ if not subfolders:
+ return
+
+ folder, get_nodes = await self.api.folder(subfolders.popleft(), folder.share)
+
+ @error_handling_wrapper
+ async def _file(self, scrape_item: ScrapeItem, file: File) -> None:
filename, ext = self.get_filename_and_ext(file.name)
- link = DOWNLOAD_URL_BASE.update_query(shared_name=shared_name, file_id=file.typed_id)
scrape_item.uploaded_at = file.date
- await self.handle_file(scrape_item.url, scrape_item, filename, ext, debrid_link=link)
-
- def build_file_system(self, items: list[Item], root_id: str) -> dict[Path, Item]:
- """Builds a flattened dictionary representing a file system from a list of items.
+ await self.handle_file(
+ scrape_item.url,
+ scrape_item,
+ file.name,
+ ext,
+ custom_filename=filename,
+ debrid_link=self.api.download(file),
+ )
+
+
+@HTTPConfig(headers={"X-Box-Client-Name": "enduserapp", "X-Box-Client-Version": "23.718.0", "Referer": str(APP_URL)})
+class BoxDotComAPI(API):
+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = APP_URL / "app-api/enduserapp"
+
+ @signature.copy(API.request_json)
+ async def request_json(self, *args, **kwargs) -> dict[str, Any]:
+ async with self.request(*args, **kwargs) as resp:
+ content = await resp.text()
+ if content.lstrip().startswith("<!DOCTYPE html>"):
+ resp.content_type = "text/html"
+ return await resp.json()
+
+ async def share(self, name: str) -> ShareItem:
+ url = (self.ENTRYPOINT / "shared-item").with_query(sharedName=name)
+ resp = await self.request_json(url)
+ return ShareItem(item_id=resp["itemID"], name=resp["sharedName"], type=resp["itemType"])
+
+ async def file(self, file_id: int, share_name: str) -> File:
+ url = (self.ENTRYPOINT / f"item/f_{file_id}").with_query(format="preview")
+ resp = await self.request_json(url, headers={"X-Box-EndUser-API": f"sharedName={share_name}"})
+ return File.from_node(_normalize_node(resp["items"][0]), share_name)
+
+ async def folder(self, folder_id: int, share_name: str) -> tuple[Folder, AsyncGenerator[map[Node]]]:
+ url = (self.ENTRYPOINT / "shared-folder").with_query(folderID=folder_id)
+ headers = {"X-Box-EndUser-API": f"sharedName={share_name}"}
+ resp = await self.request_json(url, headers=headers)
+ page_count: int = resp["pageCount"]
+
+ async def nodes() -> AsyncGenerator[map[Node]]:
+ nonlocal resp
+ for page in itertools.count(1):
+ yield map(_normalize_node, resp["items"])
+ if page >= page_count:
+ break
+ resp = await self.request_json(url.update_query(page=page), headers=headers)
+
+ return Folder.parse(resp["folder"], share_name), nodes()
+
+ def download(self, file: File) -> AbsoluteHttpURL:
+ return (APP_URL / "index.php").with_query(
+ shared_name=file.share,
+ file_id=f"f_{file.id}",
+ rm="box_download_shared_file",
+ )
+
+
+class Node(TypedDict):
+ name: str
+ type: ReadOnly[Literal["file", "folder"]]
+ id: int
+ typed_id: str
+ date: int
+ parent_id: int
- Returns:
- A 1-level dictionary where the each keys is the full path to a file/folder, and each value is the actual file/folder
- """
- path_mapping: dict[Path, Item] = {}
- parents_mapping: dict[str, list[Item]] = defaultdict(list)
[email protected](slots=True, frozen=True)
+class ShareItem:
+ item_id: int
+ name: str
+ type: Literal["file", "folder"]
- for item in items:
- parents_mapping[item.parent_id].append(item)
- def build_tree(parent_id: str, current_path: Path) -> None:
- for item in parents_mapping.get(parent_id, []):
- item_path = current_path / item.name
- path_mapping[item_path] = item
[email protected](slots=True, frozen=True)
+class File:
+ id: int
+ name: str
+ date: int
+ share: str
- if item.type == ItemType.folder:
- build_tree(item.id, item_path)
+ @classmethod
+ def from_node(cls, node: Node, share_name: str) -> Self:
+ assert node["type"] == "file"
+ return deserialize(cls, node, share=share_name)
- root_item = next(item for item in items if item.id == root_id)
- path = Path()
- path_mapping[path] = root_item
- build_tree(root_id, path)
- return dict(sorted(path_mapping.items()))
+ @property
+ def url(self) -> AbsoluteHttpURL:
+ return APP_URL / "s" / self.share / "file" / str(self.id)
-def get_canonical_url(shared_name: str, share_id: str, *, is_folder: bool = False) -> AbsoluteHttpURL:
- base = AbsoluteHttpURL(f"https://app.box.com/s/{shared_name}")
- if is_folder:
- return base / "folder" / share_id
- return base / "file" / share_id
[email protected](slots=True, frozen=True)
+class Folder:
+ id: int
+ name: str
+ share: str
+ path: PurePosixPath
+
+ @property
+ def url(self) -> AbsoluteHttpURL:
+ return APP_URL / "s" / self.share / f"folder/{self.id}"
+
+ @classmethod
+ def parse(cls, folder: dict[str, Any], share_name: str) -> Self:
+ return cls(
+ id=folder["id"],
+ name=folder["name"],
+ share=share_name,
+ path=PurePosixPath(*(p["name"] for p in folder["path"])),
+ )
+
+
+def _normalize_node(node: dict[str, Any]) -> Node:
+ return {
+ "name": node["name"],
+ "type": node["type"],
+ "id": node["id"],
+ "typed_id": node["typedID"],
+ "date": node.get("contentUpdated") or node["date"],
+ "parent_id": node["parentFolderID"],
+ }
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index e534cdff2..0e16a2684 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -892,7 +892,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
@final
@contextlib.asynccontextmanager
- async def new_task_group(self, scrape_item: ScrapeItem) -> AsyncGenerator[aio.EagerTaskGroup]:
+ async def new_task_group(self, scrape_item: ScrapeItem | AbsoluteHttpURL) -> AsyncGenerator[aio.EagerTaskGroup]:
async with aio.EagerTaskGroup() as tg:
with self.catch_errors(scrape_item):
yield tg