Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: catch errors on invividual files (Dropbox)

NTFSvolume94e85e0dbe84
diff --git a/cyberdrop_dl/crawlers/dropbox.py b/cyberdrop_dl/crawlers/dropbox.py
index 6eed85583..f65a8be26 100644
--- a/cyberdrop_dl/crawlers/dropbox.py
+++ b/cyberdrop_dl/crawlers/dropbox.py
@@ -80,21 +80,20 @@ class DropboxCrawler(Crawler):
async def file(self, scrape_item: ScrapeItem) -> None:
scrape_item.url = await self._ensure_rlkey(scrape_item.url)
async with self.request(scrape_item.url.update_query(dl=1)) as resp:
- self._file(scrape_item, resp.content_disposition.filename)
+ await self._file(scrape_item, resp.content_disposition.filename)
- def _file(self, scrape_item: ScrapeItem, filename: str) -> None:
+ @error_handling_wrapper
+ async def _file(self, scrape_item: ScrapeItem, filename: str) -> None:
scrape_item.url = view_url = scrape_item.url.with_query(rlkey=scrape_item.url.query["rlkey"], dl=0)
download_url = view_url.update_query(dl=1)
custom_filename, ext = self.get_filename_and_ext(filename)
- self.create_eager_task(
- self.handle_file(
- view_url,
- scrape_item,
- filename,
- ext,
- debrid_link=download_url,
- custom_filename=custom_filename,
- )
+ await self.handle_file(
+ view_url,
+ scrape_item,
+ filename,
+ ext,
+ debrid_link=download_url,
+ custom_filename=custom_filename,
)
async def _ensure_rlkey(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
@@ -106,6 +105,7 @@ class DropboxCrawler(Crawler):
return url
raise ScrapeError(401)
+ @error_handling_wrapper
async def _walk_folder(
self,
scrape_item: ScrapeItem,
@@ -136,7 +136,7 @@ class DropboxCrawler(Crawler):
)
continue
- self._file(new_scrape_item, node.filename)
+ self.create_eager_task(self._file(new_scrape_item, node.filename))
scrape_item.add_children()
async def _web_api_pager(
diff --git a/cyberdrop_dl/crawlers/vimeo.py b/cyberdrop_dl/crawlers/vimeo.py
index e9f1e52b5..c4d1785b7 100644
--- a/cyberdrop_dl/crawlers/vimeo.py
+++ b/cyberdrop_dl/crawlers/vimeo.py
@@ -1,8 +1,3 @@
-"""
-https://archive.org/developers/bots.html#user-agent-requirements
-https://archive.org/developers/metadata-schema/index.html#public-files-fields
-"""
-
from __future__ import annotations
import asyncio
@@ -64,7 +59,6 @@ class VimeoCrawler(Crawler):
m3u8 = info = debrid_url = None
if best.is_hls:
m3u8, info = await self.request_m3u8_playlist(best.url)
-
else:
debrid_url = best.url