Skip to content

Bulk asynchronous downloader for multiple file hosts

perf: only allow explicit URLs

NTFSvolumebaffa9dc1a09
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index 31056de35..e07bd8aeb 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -188,7 +188,7 @@ class CheveretoCrawler(Crawler, is_generic=True):
@error_handling_wrapper
async def media(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/_fluid_player.py b/cyberdrop_dl/crawlers/_fluid_player.py
index 003ba535e..5dc0e0b2d 100644
--- a/cyberdrop_dl/crawlers/_fluid_player.py
+++ b/cyberdrop_dl/crawlers/_fluid_player.py
@@ -33,7 +33,7 @@ class FluidPlayerCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.py
index c1de3ae75..155cab848 100644
--- a/cyberdrop_dl/crawlers/_kvs.py
+++ b/cyberdrop_dl/crawlers/_kvs.py
@@ -155,7 +155,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
@@ -197,7 +197,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def picture(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.py
index 81194d5aa..be2ef6f78 100644
--- a/cyberdrop_dl/crawlers/_yetishare.py
+++ b/cyberdrop_dl/crawlers/_yetishare.py
@@ -125,7 +125,7 @@ class YetiShareCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.py
index 939c54f70..e5f47ae0c 100644
--- a/cyberdrop_dl/crawlers/box_dot_com.py
+++ b/cyberdrop_dl/crawlers/box_dot_com.py
@@ -67,7 +67,7 @@ class BoxDotComCrawler(Crawler):
for trash in ("/embed_widget/", "/embed/"):
canonical_path = canonical_path.replace(trash, "")
scrape_item.url = scrape_item.url.with_path(canonical_path, keep_query=True, keep_fragment=True)
- if "file" in scrape_item.url.parts and await self.check_complete_from_referer(scrape_item):
+ if "file" in scrape_item.url.parts and await self.check_complete_from_referer(scrape_item.url):
return None
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/buzzheavier.py b/cyberdrop_dl/crawlers/buzzheavier.py
index f97d3c024..58723be83 100644
--- a/cyberdrop_dl/crawlers/buzzheavier.py
+++ b/cyberdrop_dl/crawlers/buzzheavier.py
@@ -21,7 +21,7 @@ class BuzzHeavierCrawler(Crawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
url = scrape_item.url / "download"
diff --git a/cyberdrop_dl/crawlers/cloud_mail_ru.py b/cyberdrop_dl/crawlers/cloud_mail_ru.py
index bdb9655df..13cb65238 100644
--- a/cyberdrop_dl/crawlers/cloud_mail_ru.py
+++ b/cyberdrop_dl/crawlers/cloud_mail_ru.py
@@ -55,7 +55,7 @@ class CloudMailRuCrawler(Crawler):
async def public(self, scrape_item: ScrapeItem, path: str) -> None:
node = await self._request_info(path)
if node["type"] == "file":
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
return await self._file(scrape_item, node)
diff --git a/cyberdrop_dl/crawlers/cloudflare_stream.py b/cyberdrop_dl/crawlers/cloudflare_stream.py
index 4077a8c5d..218a15f31 100644
--- a/cyberdrop_dl/crawlers/cloudflare_stream.py
+++ b/cyberdrop_dl/crawlers/cloudflare_stream.py
@@ -66,7 +66,7 @@ class CloudflareStreamCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str, token: str | None) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
m3u8_url = self.PRIMARY_URL / (token or video_id) / "manifest/video.m3u8"
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 49f230d48..ac827cf67 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -100,10 +100,6 @@ _DB_PATH_BUILDERS: MappingProxyType[str, Callable[[AbsoluteHttpURL], str]] = Map
)
-def _url(item: ScrapeItem | AbsoluteHttpURL) -> AbsoluteHttpURL:
- return item if isinstance(item, AbsoluteHttpURL) else item.url
-
-
@dataclasses.dataclass(slots=True, frozen=True, order=True)
class CrawlerInfo:
site: str
@@ -602,28 +598,26 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
@final
async def check_complete_from_referer(
- self: Crawler, scrape_item: ScrapeItem | AbsoluteHttpURL, *, any_crawler: bool = False
+ self: Crawler, referer: AbsoluteHttpURL, *, any_crawler: bool = False
) -> bool:
"""Checks if the scrape item has already been scraped.
if `any_crawler` is `True`, checks database entries for all crawlers and returns `True` if at least 1 of them has marked it as completed
"""
- url = _url(scrape_item)
domain = None if any_crawler else self.DOMAIN
- downloaded = await self.manager.database.history.check_complete_by_referer(domain, url)
+ downloaded = await self.manager.database.history.check_complete_by_referer(domain, referer)
if downloaded:
- logger.info(f"Skipping {url} as it has already been downloaded")
+ logger.info(f"Skipping {referer} as it has already been downloaded")
self.manager.scrape_mapper.tui.files.stats.previously_completed += 1
return downloaded
@final
async def check_complete_by_hash(
- self: Crawler, scrape_item: ScrapeItem | AbsoluteHttpURL, hash_type: Literal["md5", "sha256"], hash_value: str
+ self: Crawler, url: AbsoluteHttpURL, hash_type: Literal["md5", "sha256"], hash_value: str
) -> bool:
"""Returns `True` if at least 1 file with this hash is recorded on the database"""
downloaded = await self.manager.database.hash.check_hash_exists(hash_type, hash_value)
if downloaded:
- url = _url(scrape_item)
logger.info(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded")
self.manager.scrape_mapper.tui.files.stats.previously_completed += 1
return downloaded
diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.py
index 15f239334..b52e32837 100644
--- a/cyberdrop_dl/crawlers/cyberdrop.py
+++ b/cyberdrop_dl/crawlers/cyberdrop.py
@@ -66,7 +66,7 @@ class CyberdropCrawler(Crawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:
scrape_item.url = self.PRIMARY_URL / "f" / file_id
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
info, auth = await aio.safe_gather(self.api.file_info(file_id), self.api.file_auth(file_id))
diff --git a/cyberdrop_dl/crawlers/desivideo.py b/cyberdrop_dl/crawlers/desivideo.py
index ee9dd428e..3d469b07a 100644
--- a/cyberdrop_dl/crawlers/desivideo.py
+++ b/cyberdrop_dl/crawlers/desivideo.py
@@ -38,7 +38,7 @@ class DesiVideoCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.py
index 5ece238f4..66b55078b 100644
--- a/cyberdrop_dl/crawlers/efukt.py
+++ b/cyberdrop_dl/crawlers/efukt.py
@@ -77,7 +77,7 @@ class EfuktCrawler(Crawler):
@error_handling_wrapper
async def media(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
media = await self._request_media(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.py
index b7cdeb778..850129506 100644
--- a/cyberdrop_dl/crawlers/filester.py
+++ b/cyberdrop_dl/crawlers/filester.py
@@ -51,7 +51,7 @@ class FilesterCrawler(Crawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem, slug: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self._request_soup_w_pass(scrape_item.url, scrape_item.password)
diff --git a/cyberdrop_dl/crawlers/flickr.py b/cyberdrop_dl/crawlers/flickr.py
index 18ee94ec4..6eeedbeb9 100644
--- a/cyberdrop_dl/crawlers/flickr.py
+++ b/cyberdrop_dl/crawlers/flickr.py
@@ -60,7 +60,7 @@ class FlickrCrawler(Crawler):
@error_handling_wrapper
async def photo(self, scrape_item: ScrapeItem, photo_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
photo = await self.api.photo(photo_id)
diff --git a/cyberdrop_dl/crawlers/fucking_fast.py b/cyberdrop_dl/crawlers/fucking_fast.py
index 91ba83a8d..419eb48c4 100644
--- a/cyberdrop_dl/crawlers/fucking_fast.py
+++ b/cyberdrop_dl/crawlers/fucking_fast.py
@@ -29,7 +29,7 @@ class FuckingFastCrawler(Crawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.py
index 583c0e78e..63f81b2bb 100644
--- a/cyberdrop_dl/crawlers/google_drive.py
+++ b/cyberdrop_dl/crawlers/google_drive.py
@@ -176,7 +176,7 @@ class GoogleDriveCrawler(Crawler):
@error_handling_wrapper
async def _file(self, scrape_item: ScrapeItem, export_url: AbsoluteHttpURL) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
link, name = await self._get_file_info(export_url)
diff --git a/cyberdrop_dl/crawlers/gupload.py b/cyberdrop_dl/crawlers/gupload.py
index e74b07e29..a54bdb708 100644
--- a/cyberdrop_dl/crawlers/gupload.py
+++ b/cyberdrop_dl/crawlers/gupload.py
@@ -38,7 +38,7 @@ class GUploadCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
config = await self._request_video_config(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.py
index 2f186855a..1ea3654a3 100644
--- a/cyberdrop_dl/crawlers/imagebam.py
+++ b/cyberdrop_dl/crawlers/imagebam.py
@@ -92,7 +92,7 @@ class ImageBamCrawler(Crawler):
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem, soup: BeautifulSoup | None = None) -> None:
if not soup:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/imagepond.py b/cyberdrop_dl/crawlers/imagepond.py
index e39f1e0d2..905b40f89 100644
--- a/cyberdrop_dl/crawlers/imagepond.py
+++ b/cyberdrop_dl/crawlers/imagepond.py
@@ -100,7 +100,7 @@ class ImagePondCrawler(Crawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
async with self.request(scrape_item.url) as resp:
diff --git a/cyberdrop_dl/crawlers/imagevenue.py b/cyberdrop_dl/crawlers/imagevenue.py
index b38274def..b84f288a9 100644
--- a/cyberdrop_dl/crawlers/imagevenue.py
+++ b/cyberdrop_dl/crawlers/imagevenue.py
@@ -48,7 +48,7 @@ class ImageVenueCrawler(Crawler):
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
img = await self._request_img(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/imgbox.py b/cyberdrop_dl/crawlers/imgbox.py
index 9dc871345..aa0001fb6 100644
--- a/cyberdrop_dl/crawlers/imgbox.py
+++ b/cyberdrop_dl/crawlers/imgbox.py
@@ -53,7 +53,7 @@ class ImgBoxCrawler(Crawler):
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.py
index 1071bb350..e891eeca5 100644
--- a/cyberdrop_dl/crawlers/imgur.py
+++ b/cyberdrop_dl/crawlers/imgur.py
@@ -93,7 +93,7 @@ class ImgurCrawler(Crawler):
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem, image_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
image = await self._api_request("image", image_id)
await self._image(scrape_item, image)
diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.py
index 2f0528134..ef640a275 100644
--- a/cyberdrop_dl/crawlers/imx_to.py
+++ b/cyberdrop_dl/crawlers/imx_to.py
@@ -54,7 +54,7 @@ class ImxToCrawler(Crawler):
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(
diff --git a/cyberdrop_dl/crawlers/leakedzone.py b/cyberdrop_dl/crawlers/leakedzone.py
index 94a43ab78..48e23c41e 100644
--- a/cyberdrop_dl/crawlers/leakedzone.py
+++ b/cyberdrop_dl/crawlers/leakedzone.py
@@ -121,7 +121,7 @@ class LeakedZoneCrawler(Crawler):
@error_handling_wrapper
async def _video(self, scrape_item: ScrapeItem, post: Post) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
await self._handle_video(scrape_item, post)
diff --git a/cyberdrop_dl/crawlers/luxuretv.py b/cyberdrop_dl/crawlers/luxuretv.py
index 54b56224b..dd82cc2cf 100644
--- a/cyberdrop_dl/crawlers/luxuretv.py
+++ b/cyberdrop_dl/crawlers/luxuretv.py
@@ -54,7 +54,7 @@ class LuxureTVCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
soup = await self.request_soup(scrape_item.url, impersonate=True)
diff --git a/cyberdrop_dl/crawlers/masahub.py b/cyberdrop_dl/crawlers/masahub.py
index cc83f5dac..01b58ee5d 100644
--- a/cyberdrop_dl/crawlers/masahub.py
+++ b/cyberdrop_dl/crawlers/masahub.py
@@ -37,7 +37,7 @@ class MasahubCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 6543a83f2..6aed3ac5f 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -135,11 +135,11 @@ class MediaFireCrawler(Crawler, db_path="name"):
@error_handling_wrapper
async def _file(self, scrape_item: ScrapeItem, file: File, *, check_referer: bool = True) -> None:
- if check_referer and await self.check_complete_from_referer(scrape_item):
+ if check_referer and await self.check_complete_from_referer(scrape_item.url):
return
hash_algo = "sha256" if len(file.hash) == 64 else "md5"
- if await self.check_complete_by_hash(scrape_item, hash_algo, file.hash):
+ if await self.check_complete_by_hash(scrape_item.url, hash_algo, file.hash):
return
soup = await self.request_soup(scrape_item.url, impersonate=True)
diff --git a/cyberdrop_dl/crawlers/mydesi.py b/cyberdrop_dl/crawlers/mydesi.py
index 313c8ca58..148ed6c4b 100644
--- a/cyberdrop_dl/crawlers/mydesi.py
+++ b/cyberdrop_dl/crawlers/mydesi.py
@@ -43,7 +43,7 @@ class MyDesiCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/nudostartv.py b/cyberdrop_dl/crawlers/nudostartv.py
index d19e52417..5db9eb437 100644
--- a/cyberdrop_dl/crawlers/nudostartv.py
+++ b/cyberdrop_dl/crawlers/nudostartv.py
@@ -48,7 +48,7 @@ class NudoStarTVCrawler(Crawler):
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.py
index 054a30948..412f18a6f 100644
--- a/cyberdrop_dl/crawlers/onedrive.py
+++ b/cyberdrop_dl/crawlers/onedrive.py
@@ -127,7 +127,7 @@ class OneDriveCrawler(Crawler):
async def link_with_credentials(
self, scrape_item: ScrapeItem, og_share_link: AbsoluteHttpURL | None = None
) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
if og_share_link and await self.check_complete_from_referer(og_share_link):
diff --git a/cyberdrop_dl/crawlers/pimp_bunny.py b/cyberdrop_dl/crawlers/pimp_bunny.py
index d7cac9c45..b7a4bca34 100644
--- a/cyberdrop_dl/crawlers/pimp_bunny.py
+++ b/cyberdrop_dl/crawlers/pimp_bunny.py
@@ -122,7 +122,7 @@ class PimpBunnyCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.py
index b225dab06..56e17cc5a 100644
--- a/cyberdrop_dl/crawlers/pixhost.py
+++ b/cyberdrop_dl/crawlers/pixhost.py
@@ -57,7 +57,7 @@ class PixHostCrawler(Crawler):
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.py
index f15511393..d31f0287f 100644
--- a/cyberdrop_dl/crawlers/pmvhaven.py
+++ b/cyberdrop_dl/crawlers/pmvhaven.py
@@ -116,7 +116,7 @@ class PMVHavenCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
api_url = self.PRIMARY_URL / "api/videos" / video_id
diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.py
index ff4b74857..025215a57 100644
--- a/cyberdrop_dl/crawlers/pornhub.py
+++ b/cyberdrop_dl/crawlers/pornhub.py
@@ -194,7 +194,7 @@ class PornHubCrawler(Crawler):
@error_handling_wrapper
async def photo(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/postimg.py b/cyberdrop_dl/crawlers/postimg.py
index 6f8a6fe43..15a1db6cb 100644
--- a/cyberdrop_dl/crawlers/postimg.py
+++ b/cyberdrop_dl/crawlers/postimg.py
@@ -58,7 +58,7 @@ class PostImgCrawler(Crawler):
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/ranoz.py b/cyberdrop_dl/crawlers/ranoz.py
index a411b4bf2..0e30930ac 100644
--- a/cyberdrop_dl/crawlers/ranoz.py
+++ b/cyberdrop_dl/crawlers/ranoz.py
@@ -37,7 +37,7 @@ class RootzCrawler(Crawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
is_short_code = "-" not in file_id
@@ -65,7 +65,7 @@ class RanozCrawler(RootzCrawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
name, *_ = await self._request_file(self._API_ENTRYPOINT / file_id)
diff --git a/cyberdrop_dl/crawlers/realdebrid.py b/cyberdrop_dl/crawlers/realdebrid.py
index 78989364e..0400adc16 100644
--- a/cyberdrop_dl/crawlers/realdebrid.py
+++ b/cyberdrop_dl/crawlers/realdebrid.py
@@ -112,7 +112,7 @@ class RealDebridCrawler(Crawler, cdl_user_agent=True):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
url = scrape_item.url
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
title = self.create_title(f"files [{url.host}]")
diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.py
index c2d8a2611..ead5016ef 100644
--- a/cyberdrop_dl/crawlers/rumble.py
+++ b/cyberdrop_dl/crawlers/rumble.py
@@ -113,7 +113,7 @@ class RumbleCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.py
index 392c364f3..97c3548b1 100644
--- a/cyberdrop_dl/crawlers/sendvid.py
+++ b/cyberdrop_dl/crawlers/sendvid.py
@@ -35,7 +35,7 @@ class SendVidCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/soundgasm.py b/cyberdrop_dl/crawlers/soundgasm.py
index ad6641ec6..6e8a54e3d 100644
--- a/cyberdrop_dl/crawlers/soundgasm.py
+++ b/cyberdrop_dl/crawlers/soundgasm.py
@@ -40,7 +40,7 @@ class SoundGasmCrawler(Crawler):
@error_handling_wrapper
async def audio(self, scrape_item: ScrapeItem, user: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
scrape_item.setup_as_profile(self.create_title(user))
diff --git a/cyberdrop_dl/crawlers/streamtape.py b/cyberdrop_dl/crawlers/streamtape.py
index 31ec12bd5..3284f609c 100644
--- a/cyberdrop_dl/crawlers/streamtape.py
+++ b/cyberdrop_dl/crawlers/streamtape.py
@@ -36,7 +36,7 @@ class StreamtapeCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
scrape_item.url = self.PRIMARY_URL / "v" / video_id
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/tnaflix.py b/cyberdrop_dl/crawlers/tnaflix.py
index ee66d48fd..10dd7894c 100644
--- a/cyberdrop_dl/crawlers/tnaflix.py
+++ b/cyberdrop_dl/crawlers/tnaflix.py
@@ -53,7 +53,7 @@ class TNAFlixCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.py
index 0d17117b1..7a19d63da 100644
--- a/cyberdrop_dl/crawlers/tokyomotion.py
+++ b/cyberdrop_dl/crawlers/tokyomotion.py
@@ -87,7 +87,7 @@ class TokioMotionCrawler(Crawler):
@error_handling_wrapper
async def photo(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/tranny_one.py b/cyberdrop_dl/crawlers/tranny_one.py
index 9f1c65f73..5d404988f 100644
--- a/cyberdrop_dl/crawlers/tranny_one.py
+++ b/cyberdrop_dl/crawlers/tranny_one.py
@@ -54,7 +54,7 @@ class TrannyOneCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/tubecorporate/__init__.py b/cyberdrop_dl/crawlers/tubecorporate/__init__.py
index 001ededb2..a8961407f 100644
--- a/cyberdrop_dl/crawlers/tubecorporate/__init__.py
+++ b/cyberdrop_dl/crawlers/tubecorporate/__init__.py
@@ -47,7 +47,7 @@ class TubeCorporateCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
video = await self._request_video(scrape_item.url.origin(), video_id)
diff --git a/cyberdrop_dl/crawlers/turbovid.py b/cyberdrop_dl/crawlers/turbovid.py
index 06a68d6a8..df39485f9 100644
--- a/cyberdrop_dl/crawlers/turbovid.py
+++ b/cyberdrop_dl/crawlers/turbovid.py
@@ -83,12 +83,12 @@ class TurboVidCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, file_id: str) -> None:
scrape_item.url = self.PRIMARY_URL / "d" / file_id
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
checksum = css.select_text(soup, Selector.MD5)
- if await self.check_complete_by_hash(scrape_item, "md5", checksum):
+ if await self.check_complete_by_hash(scrape_item.url, "md5", checksum):
return
scrape_item.uploaded_at = self.parse_iso_date(css.select_text(soup, Selector.UPLOAD_DATE))
diff --git a/cyberdrop_dl/crawlers/twitch.py b/cyberdrop_dl/crawlers/twitch.py
index 6e4936f40..92a7ac9ef 100644
--- a/cyberdrop_dl/crawlers/twitch.py
+++ b/cyberdrop_dl/crawlers/twitch.py
@@ -91,7 +91,7 @@ class TwitchCrawler(Crawler):
async def vod(self, scrape_item: ScrapeItem, video_id: str) -> None:
video_id = video_id.removeprefix("v")
scrape_item.url = self.PRIMARY_URL / "videos" / video_id
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
video = await self.api.video(video_id)
@@ -141,7 +141,7 @@ class TwitchCrawler(Crawler):
@error_handling_wrapper
async def clip(self, scrape_item: ScrapeItem, slug: str) -> None:
scrape_item.url = _CLIPS_URL / slug
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
clip = await self.api.clip(slug)
diff --git a/cyberdrop_dl/crawlers/twpornstars.py b/cyberdrop_dl/crawlers/twpornstars.py
index 64b4b0174..a537f2c8d 100644
--- a/cyberdrop_dl/crawlers/twpornstars.py
+++ b/cyberdrop_dl/crawlers/twpornstars.py
@@ -50,7 +50,7 @@ class TwPornstarsCrawler(TwimgCrawler):
@error_handling_wrapper
async def media(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/upload_ee.py b/cyberdrop_dl/crawlers/upload_ee.py
index ecc4a2221..e623b4796 100644
--- a/cyberdrop_dl/crawlers/upload_ee.py
+++ b/cyberdrop_dl/crawlers/upload_ee.py
@@ -24,7 +24,7 @@ class UploadEECrawler(Crawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/vidara.py b/cyberdrop_dl/crawlers/vidara.py
index e13ce53fb..f93b965e0 100644
--- a/cyberdrop_dl/crawlers/vidara.py
+++ b/cyberdrop_dl/crawlers/vidara.py
@@ -35,7 +35,7 @@ class VidaraCrawler(Crawler, db_path="path_qs_frag"):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
m3u8_url, thumbnail = await self._request_stream(video_id)
diff --git a/cyberdrop_dl/crawlers/vipr_dot_im.py b/cyberdrop_dl/crawlers/vipr_dot_im.py
index cddf9ef8d..962135871 100644
--- a/cyberdrop_dl/crawlers/vipr_dot_im.py
+++ b/cyberdrop_dl/crawlers/vipr_dot_im.py
@@ -39,7 +39,7 @@ class ViprImCrawler(Crawler):
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/vsco.py b/cyberdrop_dl/crawlers/vsco.py
index cecd4c137..ca4341e23 100644
--- a/cyberdrop_dl/crawlers/vsco.py
+++ b/cyberdrop_dl/crawlers/vsco.py
@@ -40,7 +40,7 @@ class VSCOCrawler(Crawler):
@error_handling_wrapper
async def media(self, scrape_item: ScrapeItem, user: str, type_: str, id_: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
scrape_item.setup_as_profile(self.create_title(user))
@@ -96,7 +96,7 @@ class VSCOCrawler(Crawler):
m3u8 = res = None
ext = url.suffix
if ext == ".m3u8":
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
ext = ".mp4"
diff --git a/cyberdrop_dl/crawlers/wetransfer.py b/cyberdrop_dl/crawlers/wetransfer.py
index 3dd55d5ea..cb0fe52f0 100644
--- a/cyberdrop_dl/crawlers/wetransfer.py
+++ b/cyberdrop_dl/crawlers/wetransfer.py
@@ -58,7 +58,7 @@ class WeTransferCrawler(Crawler):
async def file(
self, scrape_item: ScrapeItem, file_id: str, security_hash: str, recipient_id: str | None = None
) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
payload = {"intent": "entire_transfer", "security_hash": security_hash}
diff --git a/cyberdrop_dl/crawlers/whyp_it.py b/cyberdrop_dl/crawlers/whyp_it.py
index 1d316fdfd..29fef6f9b 100644
--- a/cyberdrop_dl/crawlers/whyp_it.py
+++ b/cyberdrop_dl/crawlers/whyp_it.py
@@ -66,7 +66,7 @@ class WhypItCrawler(Crawler):
@error_handling_wrapper
async def track(self, scrape_item: ScrapeItem, track_id: str) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
track = await self.api.track(track_id, token=scrape_item.url.query.get("token"))
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.py
index 0e0c90695..8f635dde2 100644
--- a/cyberdrop_dl/crawlers/xhamster.py
+++ b/cyberdrop_dl/crawlers/xhamster.py
@@ -205,7 +205,7 @@ class XhamsterCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
initials = await self._get_window_initials(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/xxxbunker.py b/cyberdrop_dl/crawlers/xxxbunker.py
index 615f79d4b..0c7f83cb3 100644
--- a/cyberdrop_dl/crawlers/xxxbunker.py
+++ b/cyberdrop_dl/crawlers/xxxbunker.py
@@ -49,7 +49,7 @@ class XXXBunkerCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.py
index 6165e170b..c0a95e0ef 100644
--- a/cyberdrop_dl/crawlers/yandex_disk.py
+++ b/cyberdrop_dl/crawlers/yandex_disk.py
@@ -64,7 +64,7 @@ class YandexDiskCrawler(Crawler):
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
with self._request_context():
@@ -119,7 +119,7 @@ class YandexDiskCrawler(Crawler):
@error_handling_wrapper
async def _process_file(self, scrape_item: ScrapeItem, file: YandexFile) -> None:
- if await self.check_complete_from_referer(scrape_item):
+ if await self.check_complete_from_referer(scrape_item.url):
return None
referer = str(file.url)