Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: update API (GoFile)
NTFSvolumeca0e6a9b461a
diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.pyindex 2c9da6dec..62e624fd4 100644--- a/cyberdrop_dl/crawlers/gofile.py+++ b/cyberdrop_dl/crawlers/gofile.py@@ -44,10 +44,7 @@ class GoFileCrawler(Crawler):async def __async_post_init__(self) -> None:with self.catch_errors(self.api.ENTRYPOINT), self.disable_on_error("Unable create temp account"):- if key := self.config.auth.gofile.api_key:- self.api.key = key- else:- self.api.key = await self.api.create_temp_account()+ self.api.key = self.config.auth.gofile.api_key or await self.api.create_temp_account()self.update_cookies({"accountToken": self.api.key})def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:@@ -70,40 +67,44 @@ class GoFileCrawler(Crawler):return await self.folder(scrape_item, url.name, file_id)@error_handling_wrapper- async def folder(self, scrape_item: ScrapeItem, content_id: str, single_file_id: str | None = None) -> None:- node, pages = await aio.peek_first(self.api.content(content_id, scrape_item.password))- if _has_single_not_nested_file(scrape_item, node):+ async def folder(self, scrape_item: ScrapeItem, content_id: str, selected_node_id: str | None = None) -> None:+ first_node, pages = await aio.peek_first(self.api.content(content_id, scrape_item.password))+ if _has_single_not_nested_file(scrape_item, first_node):# Consider this file a loose file (autogenerated folder name)title = ""scrape_item.part_of_album = Falseelse:- title = self.create_title(name := node["name"], content_id, force_album_id=name == "root")+ title = self.create_title(+ name := first_node["name"],+ content_id,+ force_album_id=name.casefold() == "root",+ )scrape_item.part_of_album = Truescrape_item.setup_as_album(title, album_id=content_id)scrape_item.url = scrape_item.url.with_query(None)async for node in pages:- children = {node["id"]: node} if node["type"] == "file" else node["children"]+ nodes = {node["id"]: node} if node["type"] == "file" else node["children"]- if single_file_id:- file = children.get(single_file_id)- if not file:+ if selected_node_id:+ target_node = nodes.get(selected_node_id)+ if not target_node:continue- self._iter_children(scrape_item, [file])+ self._iter_nodes(scrape_item, [target_node])return- self._iter_children(scrape_item, children.values())+ self._iter_nodes(scrape_item, nodes.values())- def _iter_children(self, scrape_item: ScrapeItem, children: Iterable[Node]) -> None:+ def _iter_nodes(self, scrape_item: ScrapeItem, nodes: Iterable[Node]) -> None:def web_url(node: Node) -> AbsoluteHttpURL:node_id = node["id"]if node["type"] == "folder":return self.PRIMARY_URL / "d" / (node.get("code") or node_id)return scrape_item.url.with_fragment(node_id)- for node in children:+ for node in nodes:new_scrape_item = scrape_item.create_new(web_url(node), add_parent=True)self._handle_node(new_scrape_item, node)scrape_item.add_children()@@ -158,7 +159,7 @@ class Node(TypedDict):class File(Node):type: Literal["file"]link: str- directLink: NotRequired[str] # Only present in overloaded files (imported)+ directLink: NotRequired[str]isFrozen: NotRequired[bool] # Only present in files uploaded by free accounts and older than 30 daysviruses: NotRequired[bool]md5: str