Skip to content

Bulk asynchronous downloader for multiple file hosts

perf: checking complete from referer before request (#1028)

Barbarella66666662f2fb3a29b07
diff --git a/cyberdrop_dl/crawlers/thisvid.py b/cyberdrop_dl/crawlers/thisvid.py
index 1da8af5c0..92f39be7d 100644
--- a/cyberdrop_dl/crawlers/thisvid.py
+++ b/cyberdrop_dl/crawlers/thisvid.py
@@ -118,6 +118,9 @@ class ThisVidCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
+ if await self.check_complete_from_referer(scrape_item):
+ return
+
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)
@@ -130,11 +133,9 @@ class ThisVidCrawler(Crawler):
video = get_video_info(script.text)
title: str = soup.select_one("title").text.split("- ThisVid.com")[0].strip() # type: ignore
filename, ext = self.get_filename_and_ext(video.url.name)
- canonical_url = self.primary_base_domain / "videos" / video.id
- scrape_item.url = canonical_url
custom_filename, _ = self.get_filename_and_ext(f"{title} [{video.id}] [{video.res}]{ext}")
await self.handle_file(
- canonical_url, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=video.url
+ scrape_item.url, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=video.url
)
@error_handling_wrapper