Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: allow delayed resolution of debrid URL

NTFSvolume4cc1ba6beef2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2a0267196..0edd65065 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,12 @@ All notable changes to this project will be documented here. For more details, v
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## UNRELEASED
+
+### Fixed
+
+- Request download URL only when the download is about to begin, to prevent `403 Forbidden` errors due to expiration (Filester)
+
## [10.0.1] - 2026-06-29
### Changed
diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.py
index bd95fc8ba..a6f0265e7 100644
--- a/cyberdrop_dl/clients/downloads.py
+++ b/cyberdrop_dl/clients/downloads.py
@@ -77,10 +77,11 @@ class DownloadClient:
await asyncio.sleep(self.manager.config.downloads.total_delay)
+ download_url = await media_item.resolve()
async with self.http_client.raw_request(
- media_item.real_url,
+ download_url,
headers=media_item.headers,
- impersonate=media_item.domain in _USE_IMPERSONATION,
+ impersonate=media_item.domain in _USE_IMPERSONATION or None,
) as resp:
return await self._process_response(media_item, domain, resume_point, resp)
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 4db2bae1e..f8286f64d 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -31,7 +31,16 @@ from cyberdrop_dl.utils.errors import error_handling_context
if TYPE_CHECKING:
import datetime
import http.cookies
- from collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine, Generator, Iterable, Mapping
+ from collections.abc import (
+ AsyncGenerator,
+ AsyncIterator,
+ Awaitable,
+ Callable,
+ Coroutine,
+ Generator,
+ Iterable,
+ Mapping,
+ )
import yarl
from bs4 import BeautifulSoup, Tag
@@ -468,7 +477,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
ext: str | None = None,
*,
custom_filename: str | None = None,
- debrid_link: AbsoluteHttpURL | None = None,
+ debrid_link: Callable[[], Awaitable[AbsoluteHttpURL]] | AbsoluteHttpURL | None = None,
m3u8: m3u8.Rendition | None = None,
metadata: object = None,
referer: AbsoluteHttpURL | None = None,
diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.py
index 9e564f9be..c907fe1be 100644
--- a/cyberdrop_dl/crawlers/filester.py
+++ b/cyberdrop_dl/crawlers/filester.py
@@ -57,13 +57,15 @@ class FilesterCrawler(Crawler):
file = _parse_file(soup)
scrape_item.uploaded_at = self.parse_iso_date(file.uploaded_at)
filename, ext = self.get_filename_and_ext(file.name, mime_type=file.mime_type)
+
await self.handle_file(
scrape_item.url,
scrape_item,
file.name,
ext,
custom_filename=filename,
- debrid_link=await self.api.download(slug),
+ # Downloads expire after 30 minutes so we delay the request for it
+ debrid_link=lambda: self.api.download(slug),
)
@error_handling_wrapper
diff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.py
index 12bca4ff0..0183a6b32 100644
--- a/cyberdrop_dl/url_objects.py
+++ b/cyberdrop_dl/url_objects.py
@@ -18,7 +18,7 @@ from cyberdrop_dl.exceptions import MaxChildrenError
from cyberdrop_dl.filepath import sanitize_folder
if TYPE_CHECKING:
- from collections.abc import Generator, Iterable, Mapping, Sequence
+ from collections.abc import Awaitable, Callable, Generator, Iterable, Mapping, Sequence
@final
class AbsoluteHttpURL(yarl.URL):
@@ -134,7 +134,7 @@ class MediaItem:
download_filename: str | None = None
ext: str = ""
size: int | None = None
- debrid_url: AbsoluteHttpURL | None = None
+ debrid_url: Callable[[], Awaitable[AbsoluteHttpURL]] | AbsoluteHttpURL | None = None
duration: float | None = None
is_segment: bool = False
album_id: str | None = None
@@ -177,10 +177,20 @@ class MediaItem:
@property
def real_url(self) -> AbsoluteHttpURL:
+ if self.debrid_url and not callable(self.debrid_url):
+ return self.debrid_url
+ return self.url
+
+ async def resolve(self) -> AbsoluteHttpURL:
+ if callable(self.debrid_url):
+ self.debrid_url = await self.debrid_url()
+
return self.debrid_url or self.url
def serialize(self) -> dict[str, Any]:
me = dataclasses.asdict(self)
+ if callable(self.debrid_url):
+ me["debrid_url"] = None
if self.xxhash:
me["xxhash"] = f"xxh128:{self.xxhash}"
for name in ("is_segment",):