Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: always use x as referer for twitter media

NTFSvolumec7accc88503a
diff --git a/cyberdrop_dl/crawlers/twitter/crawler.py b/cyberdrop_dl/crawlers/twitter/crawler.py
index d33fc2941..56476bf33 100644
--- a/cyberdrop_dl/crawlers/twitter/crawler.py
+++ b/cyberdrop_dl/crawlers/twitter/crawler.py
@@ -83,7 +83,8 @@ class TwimgCrawler(Crawler):
def _prepare_media_item(self, media_item: MediaItem) -> None:
if media_item.referer.path == media_item.url.path and media_item.parents:
media_item.referer = media_item.parents[0]
- media_item.headers["Referer"] = str(media_item.referer)
+
+ media_item.headers["Referer"] = "https://x.com/"
class TwitterCrawler(Crawler):
diff --git a/cyberdrop_dl/crawlers/twpornstars.py b/cyberdrop_dl/crawlers/twpornstars.py
index 4c7987f86..91a88add2 100644
--- a/cyberdrop_dl/crawlers/twpornstars.py
+++ b/cyberdrop_dl/crawlers/twpornstars.py
@@ -68,12 +68,6 @@ class TwPornstarsCrawler(TwimgCrawler):
return
await self.photo(scrape_item, src)
- def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:
- """Prepare headers with x.com referer for video.twimg.com URLs."""
- headers = super()._prepare_headers(scrape_item)
- headers["Referer"] = "https://x.com/"
- return headers
-
@error_handling_wrapper
async def collection(self, scrape_item: ScrapeItem) -> None:
soup, pages = await aio.peek_first(self.web_pager(scrape_item.url))