Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: allow referer via URL query
NTFSvolume5245babe1ca5
diff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.pyindex 0183a6b32..c8546f5f3 100644--- a/cyberdrop_dl/url_objects.py+++ b/cyberdrop_dl/url_objects.py@@ -220,6 +220,7 @@ class ScrapeItem:parent_threads: set[AbsoluteHttpURL] = dataclasses.field(default_factory=set)max_children: Mapping[ScrapeItemType, int] | None = Nonepassword: str | None = None+ referer: AbsoluteHttpURL | None = None_children_count: int = 0_children_limit: int = 0@@ -233,7 +234,16 @@ class ScrapeItem:def from_url(cls, url: yarl.URL | str) -> Self:url = AbsoluteHttpURL(url)assert is_absolute_http_url(url)- return cls(url=url, password=url.query.get("password"))+ get = url.query.get+ try:+ referer = _url_from_query(get("referer"))+ except ValueError:+ logger.exception(+ "Unable to parse query 'referer' value (%s) from url %s , ignoring...", get("referer"), url+ )+ referer = None++ return cls(url=url, password=get("password"), referer=referer)@propertydef uploaded_at(self) -> int | None:@@ -396,3 +406,11 @@ def _extract_last_domain(folders: Sequence[str]) -> str | None:passreturn None+++def _url_from_query(query_url: str | None) -> AbsoluteHttpURL | None:+ if query_url:+ url = AbsoluteHttpURL(query_url)+ if not is_absolute_http_url(url):+ raise ValueError("URL needs to be a valid HTTP URL")+ return url