Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: allow delayed resolution of debrid URL
NTFSvolume4cc1ba6beef2
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 2a0267196..0edd65065 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -22,6 +22,12 @@ All notable changes to this project will be documented here. For more details, vThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## UNRELEASED++### Fixed++- Request download URL only when the download is about to begin, to prevent `403 Forbidden` errors due to expiration (Filester)+## [10.0.1] - 2026-06-29### Changeddiff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.pyindex bd95fc8ba..a6f0265e7 100644--- a/cyberdrop_dl/clients/downloads.py+++ b/cyberdrop_dl/clients/downloads.py@@ -77,10 +77,11 @@ class DownloadClient:await asyncio.sleep(self.manager.config.downloads.total_delay)+ download_url = await media_item.resolve()async with self.http_client.raw_request(- media_item.real_url,+ download_url,headers=media_item.headers,- impersonate=media_item.domain in _USE_IMPERSONATION,+ impersonate=media_item.domain in _USE_IMPERSONATION or None,) as resp:return await self._process_response(media_item, domain, resume_point, resp)diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 4db2bae1e..f8286f64d 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -31,7 +31,16 @@ from cyberdrop_dl.utils.errors import error_handling_contextif TYPE_CHECKING:import datetimeimport http.cookies- from collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine, Generator, Iterable, Mapping+ from collections.abc import (+ AsyncGenerator,+ AsyncIterator,+ Awaitable,+ Callable,+ Coroutine,+ Generator,+ Iterable,+ Mapping,+ )import yarlfrom bs4 import BeautifulSoup, Tag@@ -468,7 +477,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):ext: str | None = None,*,custom_filename: str | None = None,- debrid_link: AbsoluteHttpURL | None = None,+ debrid_link: Callable[[], Awaitable[AbsoluteHttpURL]] | AbsoluteHttpURL | None = None,m3u8: m3u8.Rendition | None = None,metadata: object = None,referer: AbsoluteHttpURL | None = None,diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.pyindex 9e564f9be..c907fe1be 100644--- a/cyberdrop_dl/crawlers/filester.py+++ b/cyberdrop_dl/crawlers/filester.py@@ -57,13 +57,15 @@ class FilesterCrawler(Crawler):file = _parse_file(soup)scrape_item.uploaded_at = self.parse_iso_date(file.uploaded_at)filename, ext = self.get_filename_and_ext(file.name, mime_type=file.mime_type)+await self.handle_file(scrape_item.url,scrape_item,file.name,ext,custom_filename=filename,- debrid_link=await self.api.download(slug),+ # Downloads expire after 30 minutes so we delay the request for it+ debrid_link=lambda: self.api.download(slug),)@error_handling_wrapperdiff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.pyindex 12bca4ff0..0183a6b32 100644--- a/cyberdrop_dl/url_objects.py+++ b/cyberdrop_dl/url_objects.py@@ -18,7 +18,7 @@ from cyberdrop_dl.exceptions import MaxChildrenErrorfrom cyberdrop_dl.filepath import sanitize_folderif TYPE_CHECKING:- from collections.abc import Generator, Iterable, Mapping, Sequence+ from collections.abc import Awaitable, Callable, Generator, Iterable, Mapping, Sequence@finalclass AbsoluteHttpURL(yarl.URL):@@ -134,7 +134,7 @@ class MediaItem:download_filename: str | None = Noneext: str = ""size: int | None = None- debrid_url: AbsoluteHttpURL | None = None+ debrid_url: Callable[[], Awaitable[AbsoluteHttpURL]] | AbsoluteHttpURL | None = Noneduration: float | None = Noneis_segment: bool = Falsealbum_id: str | None = None@@ -177,10 +177,20 @@ class MediaItem:@propertydef real_url(self) -> AbsoluteHttpURL:+ if self.debrid_url and not callable(self.debrid_url):+ return self.debrid_url+ return self.url++ async def resolve(self) -> AbsoluteHttpURL:+ if callable(self.debrid_url):+ self.debrid_url = await self.debrid_url()+return self.debrid_url or self.urldef serialize(self) -> dict[str, Any]:me = dataclasses.asdict(self)+ if callable(self.debrid_url):+ me["debrid_url"] = Noneif self.xxhash:me["xxhash"] = f"xxh128:{self.xxhash}"for name in ("is_segment",):