Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: only allow explicit URLs
NTFSvolumebaffa9dc1a09
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex 31056de35..e07bd8aeb 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -188,7 +188,7 @@ class CheveretoCrawler(Crawler, is_generic=True):@error_handling_wrapperasync def media(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/_fluid_player.py b/cyberdrop_dl/crawlers/_fluid_player.pyindex 003ba535e..5dc0e0b2d 100644--- a/cyberdrop_dl/crawlers/_fluid_player.py+++ b/cyberdrop_dl/crawlers/_fluid_player.py@@ -33,7 +33,7 @@ class FluidPlayerCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonesoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex c1de3ae75..155cab848 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -155,7 +155,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)@@ -197,7 +197,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def picture(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.pyindex 81194d5aa..be2ef6f78 100644--- a/cyberdrop_dl/crawlers/_yetishare.py+++ b/cyberdrop_dl/crawlers/_yetishare.py@@ -125,7 +125,7 @@ class YetiShareCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem, file_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonesoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.pyindex 939c54f70..e5f47ae0c 100644--- a/cyberdrop_dl/crawlers/box_dot_com.py+++ b/cyberdrop_dl/crawlers/box_dot_com.py@@ -67,7 +67,7 @@ class BoxDotComCrawler(Crawler):for trash in ("/embed_widget/", "/embed/"):canonical_path = canonical_path.replace(trash, "")scrape_item.url = scrape_item.url.with_path(canonical_path, keep_query=True, keep_fragment=True)- if "file" in scrape_item.url.parts and await self.check_complete_from_referer(scrape_item):+ if "file" in scrape_item.url.parts and await self.check_complete_from_referer(scrape_item.url):return Nonesoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/buzzheavier.py b/cyberdrop_dl/crawlers/buzzheavier.pyindex f97d3c024..58723be83 100644--- a/cyberdrop_dl/crawlers/buzzheavier.py+++ b/cyberdrop_dl/crawlers/buzzheavier.py@@ -21,7 +21,7 @@ class BuzzHeavierCrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnurl = scrape_item.url / "download"diff --git a/cyberdrop_dl/crawlers/cloud_mail_ru.py b/cyberdrop_dl/crawlers/cloud_mail_ru.pyindex bdb9655df..13cb65238 100644--- a/cyberdrop_dl/crawlers/cloud_mail_ru.py+++ b/cyberdrop_dl/crawlers/cloud_mail_ru.py@@ -55,7 +55,7 @@ class CloudMailRuCrawler(Crawler):async def public(self, scrape_item: ScrapeItem, path: str) -> None:node = await self._request_info(path)if node["type"] == "file":- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonereturn await self._file(scrape_item, node)diff --git a/cyberdrop_dl/crawlers/cloudflare_stream.py b/cyberdrop_dl/crawlers/cloudflare_stream.pyindex 4077a8c5d..218a15f31 100644--- a/cyberdrop_dl/crawlers/cloudflare_stream.py+++ b/cyberdrop_dl/crawlers/cloudflare_stream.py@@ -66,7 +66,7 @@ class CloudflareStreamCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str, token: str | None) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnm3u8_url = self.PRIMARY_URL / (token or video_id) / "manifest/video.m3u8"diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 49f230d48..ac827cf67 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -100,10 +100,6 @@ _DB_PATH_BUILDERS: MappingProxyType[str, Callable[[AbsoluteHttpURL], str]] = Map)-def _url(item: ScrapeItem | AbsoluteHttpURL) -> AbsoluteHttpURL:- return item if isinstance(item, AbsoluteHttpURL) else item.url--@dataclasses.dataclass(slots=True, frozen=True, order=True)class CrawlerInfo:site: str@@ -602,28 +598,26 @@ class Crawler(HTTPMixin, HLSMixin, ABC):@finalasync def check_complete_from_referer(- self: Crawler, scrape_item: ScrapeItem | AbsoluteHttpURL, *, any_crawler: bool = False+ self: Crawler, referer: AbsoluteHttpURL, *, any_crawler: bool = False) -> bool:"""Checks if the scrape item has already been scraped.if `any_crawler` is `True`, checks database entries for all crawlers and returns `True` if at least 1 of them has marked it as completed"""- url = _url(scrape_item)domain = None if any_crawler else self.DOMAIN- downloaded = await self.manager.database.history.check_complete_by_referer(domain, url)+ downloaded = await self.manager.database.history.check_complete_by_referer(domain, referer)if downloaded:- logger.info(f"Skipping {url} as it has already been downloaded")+ logger.info(f"Skipping {referer} as it has already been downloaded")self.manager.scrape_mapper.tui.files.stats.previously_completed += 1return downloaded@finalasync def check_complete_by_hash(- self: Crawler, scrape_item: ScrapeItem | AbsoluteHttpURL, hash_type: Literal["md5", "sha256"], hash_value: str+ self: Crawler, url: AbsoluteHttpURL, hash_type: Literal["md5", "sha256"], hash_value: str) -> bool:"""Returns `True` if at least 1 file with this hash is recorded on the database"""downloaded = await self.manager.database.hash.check_hash_exists(hash_type, hash_value)if downloaded:- url = _url(scrape_item)logger.info(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded")self.manager.scrape_mapper.tui.files.stats.previously_completed += 1return downloadeddiff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.pyindex 15f239334..b52e32837 100644--- a/cyberdrop_dl/crawlers/cyberdrop.py+++ b/cyberdrop_dl/crawlers/cyberdrop.py@@ -66,7 +66,7 @@ class CyberdropCrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem, file_id: str) -> None:scrape_item.url = self.PRIMARY_URL / "f" / file_id- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returninfo, auth = await aio.safe_gather(self.api.file_info(file_id), self.api.file_auth(file_id))diff --git a/cyberdrop_dl/crawlers/desivideo.py b/cyberdrop_dl/crawlers/desivideo.pyindex ee9dd428e..3d469b07a 100644--- a/cyberdrop_dl/crawlers/desivideo.py+++ b/cyberdrop_dl/crawlers/desivideo.py@@ -38,7 +38,7 @@ class DesiVideoCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonesoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.pyindex 5ece238f4..66b55078b 100644--- a/cyberdrop_dl/crawlers/efukt.py+++ b/cyberdrop_dl/crawlers/efukt.py@@ -77,7 +77,7 @@ class EfuktCrawler(Crawler):@error_handling_wrapperasync def media(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnmedia = await self._request_media(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.pyindex b7cdeb778..850129506 100644--- a/cyberdrop_dl/crawlers/filester.py+++ b/cyberdrop_dl/crawlers/filester.py@@ -51,7 +51,7 @@ class FilesterCrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem, slug: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self._request_soup_w_pass(scrape_item.url, scrape_item.password)diff --git a/cyberdrop_dl/crawlers/flickr.py b/cyberdrop_dl/crawlers/flickr.pyindex 18ee94ec4..6eeedbeb9 100644--- a/cyberdrop_dl/crawlers/flickr.py+++ b/cyberdrop_dl/crawlers/flickr.py@@ -60,7 +60,7 @@ class FlickrCrawler(Crawler):@error_handling_wrapperasync def photo(self, scrape_item: ScrapeItem, photo_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnphoto = await self.api.photo(photo_id)diff --git a/cyberdrop_dl/crawlers/fucking_fast.py b/cyberdrop_dl/crawlers/fucking_fast.pyindex 91ba83a8d..419eb48c4 100644--- a/cyberdrop_dl/crawlers/fucking_fast.py+++ b/cyberdrop_dl/crawlers/fucking_fast.py@@ -29,7 +29,7 @@ class FuckingFastCrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.pyindex 583c0e78e..63f81b2bb 100644--- a/cyberdrop_dl/crawlers/google_drive.py+++ b/cyberdrop_dl/crawlers/google_drive.py@@ -176,7 +176,7 @@ class GoogleDriveCrawler(Crawler):@error_handling_wrapperasync def _file(self, scrape_item: ScrapeItem, export_url: AbsoluteHttpURL) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnlink, name = await self._get_file_info(export_url)diff --git a/cyberdrop_dl/crawlers/gupload.py b/cyberdrop_dl/crawlers/gupload.pyindex e74b07e29..a54bdb708 100644--- a/cyberdrop_dl/crawlers/gupload.py+++ b/cyberdrop_dl/crawlers/gupload.py@@ -38,7 +38,7 @@ class GUploadCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnconfig = await self._request_video_config(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.pyindex 2f186855a..1ea3654a3 100644--- a/cyberdrop_dl/crawlers/imagebam.py+++ b/cyberdrop_dl/crawlers/imagebam.py@@ -92,7 +92,7 @@ class ImageBamCrawler(Crawler):@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem, soup: BeautifulSoup | None = None) -> None:if not soup:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/imagepond.py b/cyberdrop_dl/crawlers/imagepond.pyindex e39f1e0d2..905b40f89 100644--- a/cyberdrop_dl/crawlers/imagepond.py+++ b/cyberdrop_dl/crawlers/imagepond.py@@ -100,7 +100,7 @@ class ImagePondCrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnasync with self.request(scrape_item.url) as resp:diff --git a/cyberdrop_dl/crawlers/imagevenue.py b/cyberdrop_dl/crawlers/imagevenue.pyindex b38274def..b84f288a9 100644--- a/cyberdrop_dl/crawlers/imagevenue.py+++ b/cyberdrop_dl/crawlers/imagevenue.py@@ -48,7 +48,7 @@ class ImageVenueCrawler(Crawler):@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnimg = await self._request_img(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/imgbox.py b/cyberdrop_dl/crawlers/imgbox.pyindex 9dc871345..aa0001fb6 100644--- a/cyberdrop_dl/crawlers/imgbox.py+++ b/cyberdrop_dl/crawlers/imgbox.py@@ -53,7 +53,7 @@ class ImgBoxCrawler(Crawler):@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.pyindex 1071bb350..e891eeca5 100644--- a/cyberdrop_dl/crawlers/imgur.py+++ b/cyberdrop_dl/crawlers/imgur.py@@ -93,7 +93,7 @@ class ImgurCrawler(Crawler):@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem, image_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnimage = await self._api_request("image", image_id)await self._image(scrape_item, image)diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.pyindex 2f0528134..ef640a275 100644--- a/cyberdrop_dl/crawlers/imx_to.py+++ b/cyberdrop_dl/crawlers/imx_to.py@@ -54,7 +54,7 @@ class ImxToCrawler(Crawler):@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(diff --git a/cyberdrop_dl/crawlers/leakedzone.py b/cyberdrop_dl/crawlers/leakedzone.pyindex 94a43ab78..48e23c41e 100644--- a/cyberdrop_dl/crawlers/leakedzone.py+++ b/cyberdrop_dl/crawlers/leakedzone.py@@ -121,7 +121,7 @@ class LeakedZoneCrawler(Crawler):@error_handling_wrapperasync def _video(self, scrape_item: ScrapeItem, post: Post) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnawait self._handle_video(scrape_item, post)diff --git a/cyberdrop_dl/crawlers/luxuretv.py b/cyberdrop_dl/crawlers/luxuretv.pyindex 54b56224b..dd82cc2cf 100644--- a/cyberdrop_dl/crawlers/luxuretv.py+++ b/cyberdrop_dl/crawlers/luxuretv.py@@ -54,7 +54,7 @@ class LuxureTVCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonesoup = await self.request_soup(scrape_item.url, impersonate=True)diff --git a/cyberdrop_dl/crawlers/masahub.py b/cyberdrop_dl/crawlers/masahub.pyindex cc83f5dac..01b58ee5d 100644--- a/cyberdrop_dl/crawlers/masahub.py+++ b/cyberdrop_dl/crawlers/masahub.py@@ -37,7 +37,7 @@ class MasahubCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonesoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 6543a83f2..6aed3ac5f 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -135,11 +135,11 @@ class MediaFireCrawler(Crawler, db_path="name"):@error_handling_wrapperasync def _file(self, scrape_item: ScrapeItem, file: File, *, check_referer: bool = True) -> None:- if check_referer and await self.check_complete_from_referer(scrape_item):+ if check_referer and await self.check_complete_from_referer(scrape_item.url):returnhash_algo = "sha256" if len(file.hash) == 64 else "md5"- if await self.check_complete_by_hash(scrape_item, hash_algo, file.hash):+ if await self.check_complete_by_hash(scrape_item.url, hash_algo, file.hash):returnsoup = await self.request_soup(scrape_item.url, impersonate=True)diff --git a/cyberdrop_dl/crawlers/mydesi.py b/cyberdrop_dl/crawlers/mydesi.pyindex 313c8ca58..148ed6c4b 100644--- a/cyberdrop_dl/crawlers/mydesi.py+++ b/cyberdrop_dl/crawlers/mydesi.py@@ -43,7 +43,7 @@ class MyDesiCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonesoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/nudostartv.py b/cyberdrop_dl/crawlers/nudostartv.pyindex d19e52417..5db9eb437 100644--- a/cyberdrop_dl/crawlers/nudostartv.py+++ b/cyberdrop_dl/crawlers/nudostartv.py@@ -48,7 +48,7 @@ class NudoStarTVCrawler(Crawler):@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.pyindex 054a30948..412f18a6f 100644--- a/cyberdrop_dl/crawlers/onedrive.py+++ b/cyberdrop_dl/crawlers/onedrive.py@@ -127,7 +127,7 @@ class OneDriveCrawler(Crawler):async def link_with_credentials(self, scrape_item: ScrapeItem, og_share_link: AbsoluteHttpURL | None = None) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnif og_share_link and await self.check_complete_from_referer(og_share_link):diff --git a/cyberdrop_dl/crawlers/pimp_bunny.py b/cyberdrop_dl/crawlers/pimp_bunny.pyindex d7cac9c45..b7a4bca34 100644--- a/cyberdrop_dl/crawlers/pimp_bunny.py+++ b/cyberdrop_dl/crawlers/pimp_bunny.py@@ -122,7 +122,7 @@ class PimpBunnyCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.pyindex b225dab06..56e17cc5a 100644--- a/cyberdrop_dl/crawlers/pixhost.py+++ b/cyberdrop_dl/crawlers/pixhost.py@@ -57,7 +57,7 @@ class PixHostCrawler(Crawler):@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.pyindex f15511393..d31f0287f 100644--- a/cyberdrop_dl/crawlers/pmvhaven.py+++ b/cyberdrop_dl/crawlers/pmvhaven.py@@ -116,7 +116,7 @@ class PMVHavenCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnapi_url = self.PRIMARY_URL / "api/videos" / video_iddiff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.pyindex ff4b74857..025215a57 100644--- a/cyberdrop_dl/crawlers/pornhub.py+++ b/cyberdrop_dl/crawlers/pornhub.py@@ -194,7 +194,7 @@ class PornHubCrawler(Crawler):@error_handling_wrapperasync def photo(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/postimg.py b/cyberdrop_dl/crawlers/postimg.pyindex 6f8a6fe43..15a1db6cb 100644--- a/cyberdrop_dl/crawlers/postimg.py+++ b/cyberdrop_dl/crawlers/postimg.py@@ -58,7 +58,7 @@ class PostImgCrawler(Crawler):@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/ranoz.py b/cyberdrop_dl/crawlers/ranoz.pyindex a411b4bf2..0e30930ac 100644--- a/cyberdrop_dl/crawlers/ranoz.py+++ b/cyberdrop_dl/crawlers/ranoz.py@@ -37,7 +37,7 @@ class RootzCrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem, file_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnis_short_code = "-" not in file_id@@ -65,7 +65,7 @@ class RanozCrawler(RootzCrawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem, file_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnname, *_ = await self._request_file(self._API_ENTRYPOINT / file_id)diff --git a/cyberdrop_dl/crawlers/realdebrid.py b/cyberdrop_dl/crawlers/realdebrid.pyindex 78989364e..0400adc16 100644--- a/cyberdrop_dl/crawlers/realdebrid.py+++ b/cyberdrop_dl/crawlers/realdebrid.py@@ -112,7 +112,7 @@ class RealDebridCrawler(Crawler, cdl_user_agent=True):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:url = scrape_item.url- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returntitle = self.create_title(f"files [{url.host}]")diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.pyindex c2d8a2611..ead5016ef 100644--- a/cyberdrop_dl/crawlers/rumble.py+++ b/cyberdrop_dl/crawlers/rumble.py@@ -113,7 +113,7 @@ class RumbleCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.pyindex 392c364f3..97c3548b1 100644--- a/cyberdrop_dl/crawlers/sendvid.py+++ b/cyberdrop_dl/crawlers/sendvid.py@@ -35,7 +35,7 @@ class SendVidCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/soundgasm.py b/cyberdrop_dl/crawlers/soundgasm.pyindex ad6641ec6..6e8a54e3d 100644--- a/cyberdrop_dl/crawlers/soundgasm.py+++ b/cyberdrop_dl/crawlers/soundgasm.py@@ -40,7 +40,7 @@ class SoundGasmCrawler(Crawler):@error_handling_wrapperasync def audio(self, scrape_item: ScrapeItem, user: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnscrape_item.setup_as_profile(self.create_title(user))diff --git a/cyberdrop_dl/crawlers/streamtape.py b/cyberdrop_dl/crawlers/streamtape.pyindex 31ec12bd5..3284f609c 100644--- a/cyberdrop_dl/crawlers/streamtape.py+++ b/cyberdrop_dl/crawlers/streamtape.py@@ -36,7 +36,7 @@ class StreamtapeCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:scrape_item.url = self.PRIMARY_URL / "v" / video_id- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonesoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/tnaflix.py b/cyberdrop_dl/crawlers/tnaflix.pyindex ee66d48fd..10dd7894c 100644--- a/cyberdrop_dl/crawlers/tnaflix.py+++ b/cyberdrop_dl/crawlers/tnaflix.py@@ -53,7 +53,7 @@ class TNAFlixCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonesoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.pyindex 0d17117b1..7a19d63da 100644--- a/cyberdrop_dl/crawlers/tokyomotion.py+++ b/cyberdrop_dl/crawlers/tokyomotion.py@@ -87,7 +87,7 @@ class TokioMotionCrawler(Crawler):@error_handling_wrapperasync def photo(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/tranny_one.py b/cyberdrop_dl/crawlers/tranny_one.pyindex 9f1c65f73..5d404988f 100644--- a/cyberdrop_dl/crawlers/tranny_one.py+++ b/cyberdrop_dl/crawlers/tranny_one.py@@ -54,7 +54,7 @@ class TrannyOneCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonesoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/tubecorporate/__init__.py b/cyberdrop_dl/crawlers/tubecorporate/__init__.pyindex 001ededb2..a8961407f 100644--- a/cyberdrop_dl/crawlers/tubecorporate/__init__.py+++ b/cyberdrop_dl/crawlers/tubecorporate/__init__.py@@ -47,7 +47,7 @@ class TubeCorporateCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonevideo = await self._request_video(scrape_item.url.origin(), video_id)diff --git a/cyberdrop_dl/crawlers/turbovid.py b/cyberdrop_dl/crawlers/turbovid.pyindex 06a68d6a8..df39485f9 100644--- a/cyberdrop_dl/crawlers/turbovid.py+++ b/cyberdrop_dl/crawlers/turbovid.py@@ -83,12 +83,12 @@ class TurboVidCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, file_id: str) -> None:scrape_item.url = self.PRIMARY_URL / "d" / file_id- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)checksum = css.select_text(soup, Selector.MD5)- if await self.check_complete_by_hash(scrape_item, "md5", checksum):+ if await self.check_complete_by_hash(scrape_item.url, "md5", checksum):returnscrape_item.uploaded_at = self.parse_iso_date(css.select_text(soup, Selector.UPLOAD_DATE))diff --git a/cyberdrop_dl/crawlers/twitch.py b/cyberdrop_dl/crawlers/twitch.pyindex 6e4936f40..92a7ac9ef 100644--- a/cyberdrop_dl/crawlers/twitch.py+++ b/cyberdrop_dl/crawlers/twitch.py@@ -91,7 +91,7 @@ class TwitchCrawler(Crawler):async def vod(self, scrape_item: ScrapeItem, video_id: str) -> None:video_id = video_id.removeprefix("v")scrape_item.url = self.PRIMARY_URL / "videos" / video_id- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnvideo = await self.api.video(video_id)@@ -141,7 +141,7 @@ class TwitchCrawler(Crawler):@error_handling_wrapperasync def clip(self, scrape_item: ScrapeItem, slug: str) -> None:scrape_item.url = _CLIPS_URL / slug- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnclip = await self.api.clip(slug)diff --git a/cyberdrop_dl/crawlers/twpornstars.py b/cyberdrop_dl/crawlers/twpornstars.pyindex 64b4b0174..a537f2c8d 100644--- a/cyberdrop_dl/crawlers/twpornstars.py+++ b/cyberdrop_dl/crawlers/twpornstars.py@@ -50,7 +50,7 @@ class TwPornstarsCrawler(TwimgCrawler):@error_handling_wrapperasync def media(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/upload_ee.py b/cyberdrop_dl/crawlers/upload_ee.pyindex ecc4a2221..e623b4796 100644--- a/cyberdrop_dl/crawlers/upload_ee.py+++ b/cyberdrop_dl/crawlers/upload_ee.py@@ -24,7 +24,7 @@ class UploadEECrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/vidara.py b/cyberdrop_dl/crawlers/vidara.pyindex e13ce53fb..f93b965e0 100644--- a/cyberdrop_dl/crawlers/vidara.py+++ b/cyberdrop_dl/crawlers/vidara.py@@ -35,7 +35,7 @@ class VidaraCrawler(Crawler, db_path="path_qs_frag"):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnm3u8_url, thumbnail = await self._request_stream(video_id)diff --git a/cyberdrop_dl/crawlers/vipr_dot_im.py b/cyberdrop_dl/crawlers/vipr_dot_im.pyindex cddf9ef8d..962135871 100644--- a/cyberdrop_dl/crawlers/vipr_dot_im.py+++ b/cyberdrop_dl/crawlers/vipr_dot_im.py@@ -39,7 +39,7 @@ class ViprImCrawler(Crawler):@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/vsco.py b/cyberdrop_dl/crawlers/vsco.pyindex cecd4c137..ca4341e23 100644--- a/cyberdrop_dl/crawlers/vsco.py+++ b/cyberdrop_dl/crawlers/vsco.py@@ -40,7 +40,7 @@ class VSCOCrawler(Crawler):@error_handling_wrapperasync def media(self, scrape_item: ScrapeItem, user: str, type_: str, id_: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnscrape_item.setup_as_profile(self.create_title(user))@@ -96,7 +96,7 @@ class VSCOCrawler(Crawler):m3u8 = res = Noneext = url.suffixif ext == ".m3u8":- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnext = ".mp4"diff --git a/cyberdrop_dl/crawlers/wetransfer.py b/cyberdrop_dl/crawlers/wetransfer.pyindex 3dd55d5ea..cb0fe52f0 100644--- a/cyberdrop_dl/crawlers/wetransfer.py+++ b/cyberdrop_dl/crawlers/wetransfer.py@@ -58,7 +58,7 @@ class WeTransferCrawler(Crawler):async def file(self, scrape_item: ScrapeItem, file_id: str, security_hash: str, recipient_id: str | None = None) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnpayload = {"intent": "entire_transfer", "security_hash": security_hash}diff --git a/cyberdrop_dl/crawlers/whyp_it.py b/cyberdrop_dl/crawlers/whyp_it.pyindex 1d316fdfd..29fef6f9b 100644--- a/cyberdrop_dl/crawlers/whyp_it.py+++ b/cyberdrop_dl/crawlers/whyp_it.py@@ -66,7 +66,7 @@ class WhypItCrawler(Crawler):@error_handling_wrapperasync def track(self, scrape_item: ScrapeItem, track_id: str) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returntrack = await self.api.track(track_id, token=scrape_item.url.query.get("token"))diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex 0e0c90695..8f635dde2 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -205,7 +205,7 @@ class XhamsterCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returninitials = await self._get_window_initials(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/xxxbunker.py b/cyberdrop_dl/crawlers/xxxbunker.pyindex 615f79d4b..0c7f83cb3 100644--- a/cyberdrop_dl/crawlers/xxxbunker.py+++ b/cyberdrop_dl/crawlers/xxxbunker.py@@ -49,7 +49,7 @@ class XXXBunkerCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.pyindex 6165e170b..c0a95e0ef 100644--- a/cyberdrop_dl/crawlers/yandex_disk.py+++ b/cyberdrop_dl/crawlers/yandex_disk.py@@ -64,7 +64,7 @@ class YandexDiskCrawler(Crawler):@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonewith self._request_context():@@ -119,7 +119,7 @@ class YandexDiskCrawler(Crawler):@error_handling_wrapperasync def _process_file(self, scrape_item: ScrapeItem, file: YandexFile) -> None:- if await self.check_complete_from_referer(scrape_item):+ if await self.check_complete_from_referer(scrape_item.url):return Nonereferer = str(file.url)