Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: always use x as referer for twitter media
NTFSvolumec7accc88503a
diff --git a/cyberdrop_dl/crawlers/twitter/crawler.py b/cyberdrop_dl/crawlers/twitter/crawler.pyindex d33fc2941..56476bf33 100644--- a/cyberdrop_dl/crawlers/twitter/crawler.py+++ b/cyberdrop_dl/crawlers/twitter/crawler.py@@ -83,7 +83,8 @@ class TwimgCrawler(Crawler):def _prepare_media_item(self, media_item: MediaItem) -> None:if media_item.referer.path == media_item.url.path and media_item.parents:media_item.referer = media_item.parents[0]- media_item.headers["Referer"] = str(media_item.referer)++ media_item.headers["Referer"] = "https://x.com/"class TwitterCrawler(Crawler):diff --git a/cyberdrop_dl/crawlers/twpornstars.py b/cyberdrop_dl/crawlers/twpornstars.pyindex 4c7987f86..91a88add2 100644--- a/cyberdrop_dl/crawlers/twpornstars.py+++ b/cyberdrop_dl/crawlers/twpornstars.py@@ -68,12 +68,6 @@ class TwPornstarsCrawler(TwimgCrawler):returnawait self.photo(scrape_item, src)- def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:- """Prepare headers with x.com referer for video.twimg.com URLs."""- headers = super()._prepare_headers(scrape_item)- headers["Referer"] = "https://x.com/"- return headers-@error_handling_wrapperasync def collection(self, scrape_item: ScrapeItem) -> None:soup, pages = await aio.peek_first(self.web_pager(scrape_item.url))