Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: checking complete from referer before request (#1028)
Barbarella66666662f2fb3a29b07
diff --git a/cyberdrop_dl/crawlers/thisvid.py b/cyberdrop_dl/crawlers/thisvid.pyindex 1da8af5c0..92f39be7d 100644--- a/cyberdrop_dl/crawlers/thisvid.py+++ b/cyberdrop_dl/crawlers/thisvid.py@@ -118,6 +118,9 @@ class ThisVidCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:+ if await self.check_complete_from_referer(scrape_item):+ return+async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)@@ -130,11 +133,9 @@ class ThisVidCrawler(Crawler):video = get_video_info(script.text)title: str = soup.select_one("title").text.split("- ThisVid.com")[0].strip() # type: ignorefilename, ext = self.get_filename_and_ext(video.url.name)- canonical_url = self.primary_base_domain / "videos" / video.id- scrape_item.url = canonical_urlcustom_filename, _ = self.get_filename_and_ext(f"{title} [{video.id}] [{video.res}]{ext}")await self.handle_file(- canonical_url, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=video.url+ scrape_item.url, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=video.url)@error_handling_wrapper