Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove redundant returns in fetch

NTFSvolume278c8342f198
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index ff815fd0a..3d1093816 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -70,15 +70,15 @@ class CheveretoCrawler(Crawler, is_generic=True):
match scrape_item.url.parts[1:]:
case ["a" | "album" | "category", album_slug]:
- return await self.album(scrape_item, _id(album_slug))
+ await self.album(scrape_item, _id(album_slug))
case ["img" | "image" | "video" | "videos", _]:
- return await self.media(scrape_item)
+ await self.media(scrape_item)
case ["images", _, *_]:
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
case [_, "albums"]:
- return await self.profile(scrape_item, albums=True)
+ await self.profile(scrape_item, albums=True)
case [_]:
- return await self.profile(scrape_item)
+ await self.profile(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.py
index 042a3547d..42a6e9a6d 100644
--- a/cyberdrop_dl/crawlers/_kvs.py
+++ b/cyberdrop_dl/crawlers/_kvs.py
@@ -83,22 +83,22 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
return url / ""
return url
- async def fetch(self, scrape_item: ScrapeItem) -> None: # noqa: PLR0911
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["categories" | "tags" as type_, name]:
- return await self.collection(scrape_item, name, type_)
+ await self.collection(scrape_item, name, type_)
case ["search", query]:
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case ["members", member_id, "public_videos" | "favourite_videos" | "private_videos", *_]:
- return await self.profile(scrape_item, member_id, entire_profile=False)
+ await self.profile(scrape_item, member_id, entire_profile=False)
case ["members", member_id, *_]:
- return await self.profile(scrape_item, member_id)
+ await self.profile(scrape_item, member_id)
case ["videos" | "video", _, *_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case ["albums" | "album", _]:
- return await self.album(scrape_item)
+ await self.album(scrape_item)
case ["albums" | "album", _, _, *_]:
- return await self.picture(scrape_item)
+ await self.picture(scrape_item)
case _:
if query := scrape_item.url.query.get("q"):
return await self.search(scrape_item, query)
diff --git a/cyberdrop_dl/crawlers/_xfs.py b/cyberdrop_dl/crawlers/_xfs.py
index 11a94aecf..4f67e5412 100644
--- a/cyberdrop_dl/crawlers/_xfs.py
+++ b/cyberdrop_dl/crawlers/_xfs.py
@@ -112,6 +112,6 @@ class EmbedOnlyMixin(_XVSMixinBase):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["embed", video_id]:
- return await self.embed(scrape_item, video_id)
+ await self.embed(scrape_item, video_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.py
index cd42cddd1..38c40884c 100644
--- a/cyberdrop_dl/crawlers/_yetishare.py
+++ b/cyberdrop_dl/crawlers/_yetishare.py
@@ -58,13 +58,13 @@ class YetiShareCrawler(Crawler, is_abc=True):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["folder", folder_id, *_]:
- return await self.folder(scrape_item, folder_id)
+ await self.folder(scrape_item, folder_id)
case ["shared", folder_id]:
- return await self.folder(scrape_item, folder_id, is_shared=True)
+ await self.folder(scrape_item, folder_id, is_shared=True)
case [file_id]:
- return await self.file(scrape_item, file_id)
+ await self.file(scrape_item, file_id)
case [file_id, _]:
- return await self.file(scrape_item, file_id)
+ await self.file(scrape_item, file_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/acast.py b/cyberdrop_dl/crawlers/acast.py
index 0ea5741f4..d2071455b 100644
--- a/cyberdrop_dl/crawlers/acast.py
+++ b/cyberdrop_dl/crawlers/acast.py
@@ -23,9 +23,9 @@ class ACastCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [show, "episodes", episode_id]:
- return await self.episode(scrape_item, show, episode_id)
+ await self.episode(scrape_item, show, episode_id)
case [show]:
- return await self.show(scrape_item, show)
+ await self.show(scrape_item, show)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/adobe_lightroom.py b/cyberdrop_dl/crawlers/adobe_lightroom.py
index a09b5fe0b..ec890fce4 100644
--- a/cyberdrop_dl/crawlers/adobe_lightroom.py
+++ b/cyberdrop_dl/crawlers/adobe_lightroom.py
@@ -30,7 +30,7 @@ class AdobeLightroomCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["shares", space_id]:
- return await self.album(scrape_item, space_id)
+ await self.album(scrape_item, space_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/anysex.py b/cyberdrop_dl/crawlers/anysex.py
index 48e046f53..fad2b4df6 100644
--- a/cyberdrop_dl/crawlers/anysex.py
+++ b/cyberdrop_dl/crawlers/anysex.py
@@ -31,17 +31,17 @@ class AnySexCrawler(FluidPlayerCrawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [*_, "video", video_id, _]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["contents", _, *_]:
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
case ["photos", "search", *_] if query := scrape_item.url.query.get("q"):
query = query.replace("-", " ")
- return await self.photo_search(scrape_item, query)
+ await self.photo_search(scrape_item, query)
case [*_, "photos", album_id, _]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case ["search" as type_, *_] if query := scrape_item.url.query.get("q"):
query = query.replace("-", " ")
- return await self.collection(scrape_item, type_, query)
+ await self.collection(scrape_item, type_, query)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/apkmirror.py b/cyberdrop_dl/crawlers/apkmirror.py
index 851ecc548..f554f188f 100644
--- a/cyberdrop_dl/crawlers/apkmirror.py
+++ b/cyberdrop_dl/crawlers/apkmirror.py
@@ -32,7 +32,7 @@ class APKMirrorCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["apk", _, _, _, variant, *_] if variant.endswith("-download"):
- return await self.apk(scrape_item)
+ await self.apk(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/archive_org.py b/cyberdrop_dl/crawlers/archive_org.py
index 2a704931c..fb5bcc6cb 100644
--- a/cyberdrop_dl/crawlers/archive_org.py
+++ b/cyberdrop_dl/crawlers/archive_org.py
@@ -47,7 +47,7 @@ class ArchiveOrgCrawler(Crawler):
match scrape_item.url.parts[1:]:
case ["details" | "download", identifier, *rest]:
base_path = "/".join(rest) if rest else None
- return await self.item(scrape_item, identifier, base_path)
+ await self.item(scrape_item, identifier, base_path)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/archivebate.py b/cyberdrop_dl/crawlers/archivebate.py
index b4e83b907..c283e81eb 100644
--- a/cyberdrop_dl/crawlers/archivebate.py
+++ b/cyberdrop_dl/crawlers/archivebate.py
@@ -33,7 +33,7 @@ class ArchiveBateCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["watch", _]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/bandcamp.py b/cyberdrop_dl/crawlers/bandcamp.py
index 16ce8314e..6c85b91ec 100644
--- a/cyberdrop_dl/crawlers/bandcamp.py
+++ b/cyberdrop_dl/crawlers/bandcamp.py
@@ -43,9 +43,9 @@ class BandcampCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["track", _, *_]:
- return await self.song(scrape_item)
+ await self.song(scrape_item)
case ["album", _, *_]:
- return await self.album(scrape_item)
+ await self.album(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/beeg.py b/cyberdrop_dl/crawlers/beeg.py
index 60b6ca41f..204cb2544 100644
--- a/cyberdrop_dl/crawlers/beeg.py
+++ b/cyberdrop_dl/crawlers/beeg.py
@@ -41,10 +41,7 @@ class BeegComCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
- case ["reacted", "saved", "following"]:
- raise ValueError
-
- case [slug]:
+ case [slug] if slug not in {"reacted", "saved", "following"}:
if slug.startswith("-"):
video_id = int(slug.lstrip("-"))
await self.video(scrape_item, video_id)
diff --git a/cyberdrop_dl/crawlers/bunkr.py b/cyberdrop_dl/crawlers/bunkr.py
index 997445fbb..652a7f521 100644
--- a/cyberdrop_dl/crawlers/bunkr.py
+++ b/cyberdrop_dl/crawlers/bunkr.py
@@ -94,15 +94,15 @@ class BunkrCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["file", file_id] if scrape_item.url.host == self.api.DL_ENDPOINT.host:
- return await self.file_download(scrape_item, file_id)
+ await self.file_download(scrape_item, file_id)
case ["a", album_id]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case ["v" | "d" | "i", _]:
- return await self.follow_redirect(scrape_item)
+ await self.follow_redirect(scrape_item)
case ["f", _]:
- return await self.file(scrape_item)
+ await self.file(scrape_item)
case [_] if _is_stream_redirect(scrape_item.url.host):
- return await self.follow_redirect(scrape_item)
+ await self.follow_redirect(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/bunkr_albums.py b/cyberdrop_dl/crawlers/bunkr_albums.py
index 84ae5d732..b2ed99da3 100644
--- a/cyberdrop_dl/crawlers/bunkr_albums.py
+++ b/cyberdrop_dl/crawlers/bunkr_albums.py
@@ -24,7 +24,7 @@ class BunkrAlbumsCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [] | [""] if query := scrape_item.url.query.get("search"):
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/camwhores_dot_tv.py b/cyberdrop_dl/crawlers/camwhores_dot_tv.py
index 6eeacb2da..99bb3db31 100644
--- a/cyberdrop_dl/crawlers/camwhores_dot_tv.py
+++ b/cyberdrop_dl/crawlers/camwhores_dot_tv.py
@@ -30,11 +30,11 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler, ensure_trailing_slash=True):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:-1]:
case ["categories" | "tags" as type_, _]:
- return await self.search(scrape_item, type_)
+ await self.search(scrape_item, type_)
case ["search" as type_, query]:
- return await self.search(scrape_item, type_, query)
+ await self.search(scrape_item, type_, query)
case ["videos", _, _]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/cara_app.py b/cyberdrop_dl/crawlers/cara_app.py
index 378424246..77a6f3c44 100644
--- a/cyberdrop_dl/crawlers/cara_app.py
+++ b/cyberdrop_dl/crawlers/cara_app.py
@@ -37,9 +37,9 @@ class CaraCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["post", user_name]:
- return await self.post(scrape_item, user_name)
+ await self.post(scrape_item, user_name)
case [user_name]:
- return await self.user(scrape_item, user_name)
+ await self.user(scrape_item, user_name)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/catbox.py b/cyberdrop_dl/crawlers/catbox.py
index 73dce5ce3..fd179ed0b 100644
--- a/cyberdrop_dl/crawlers/catbox.py
+++ b/cyberdrop_dl/crawlers/catbox.py
@@ -9,17 +9,19 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
-_CDN_HOSTS = "litter.catbox.moe", "files.catbox.moe", "litter.fatbox.moe", "files.fatbox.moe"
-
-
class CatboxCrawler(Crawler):
- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = _CDN_HOSTS
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (
+ "litter.catbox.moe",
+ "files.catbox.moe",
+ "litter.fatbox.moe",
+ "files.fatbox.moe",
+ )
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Direct links": ""}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://catbox.moe")
DOMAIN: ClassVar[str] = "catbox.moe"
FOLDER_DOMAIN: ClassVar[str] = "Catbox"
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if scrape_item.url.host in _CDN_HOSTS:
- return await self.direct_file(scrape_item, assume_ext=".zip")
- raise ValueError
+ if scrape_item.url.host not in self.SUPPORTED_DOMAINS:
+ raise ValueError
+ await self.direct_file(scrape_item, assume_ext=".zip")
diff --git a/cyberdrop_dl/crawlers/cloud_mail_ru.py b/cyberdrop_dl/crawlers/cloud_mail_ru.py
index 0e2b8a2e2..1e6fe5791 100644
--- a/cyberdrop_dl/crawlers/cloud_mail_ru.py
+++ b/cyberdrop_dl/crawlers/cloud_mail_ru.py
@@ -21,7 +21,7 @@ class CloudMailRuCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["public", *rest] if rest:
- return await self.public(scrape_item, path="/".join(rest))
+ await self.public(scrape_item, path="/".join(rest))
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/cloudflare_stream.py b/cyberdrop_dl/crawlers/cloudflare_stream.py
index a1d1b1520..5f21b4499 100644
--- a/cyberdrop_dl/crawlers/cloudflare_stream.py
+++ b/cyberdrop_dl/crawlers/cloudflare_stream.py
@@ -59,10 +59,10 @@ class CloudflareStreamCrawler(Crawler):
video_id = jwt.payload["sub"]
if jwt.is_expired():
self.raise_exc(scrape_item, ScrapeError(401, "Access token to the video has expired"))
- return None
+ return
scrape_item.url = _DEFAULT_VIDEO_CDN / video_id
- return await self.video(scrape_item, video_id, token)
+ await self.video(scrape_item, video_id, token)
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str, token: str | None) -> None:
diff --git a/cyberdrop_dl/crawlers/clyp_it.py b/cyberdrop_dl/crawlers/clyp_it.py
index ce54bf543..657b5a9d4 100644
--- a/cyberdrop_dl/crawlers/clyp_it.py
+++ b/cyberdrop_dl/crawlers/clyp_it.py
@@ -32,9 +32,9 @@ class ClypItCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [audio_id]:
- return await self.audio(scrape_item, audio_id)
+ await self.audio(scrape_item, audio_id)
case ["user", user_id]:
- return await self.user(scrape_item, user_id)
+ await self.user(scrape_item, user_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/crazyshit.py b/cyberdrop_dl/crawlers/crazyshit.py
index 11738f1e2..99064fb81 100644
--- a/cyberdrop_dl/crawlers/crazyshit.py
+++ b/cyberdrop_dl/crawlers/crazyshit.py
@@ -25,9 +25,9 @@ class CrazyShitCrawler(Crawler):
match scrape_item.url.parts[1:]:
case ["cnt", "medias", slug]:
video_id = str(int(slug.partition("-")[0]))
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["series", _]:
- return await self.series(scrape_item)
+ await self.series(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/daftporn.py b/cyberdrop_dl/crawlers/daftporn.py
index 18c3b7bd5..61815b560 100644
--- a/cyberdrop_dl/crawlers/daftporn.py
+++ b/cyberdrop_dl/crawlers/daftporn.py
@@ -21,7 +21,7 @@ class DaftPornCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["extreme-videos", _]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/dailymotion.py b/cyberdrop_dl/crawlers/dailymotion.py
index a38400481..383f81981 100644
--- a/cyberdrop_dl/crawlers/dailymotion.py
+++ b/cyberdrop_dl/crawlers/dailymotion.py
@@ -58,9 +58,9 @@ class DailyMotionCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["video", video_id]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["playlist", slug]:
- return await self.playlist(scrape_item, slug)
+ await self.playlist(scrape_item, slug)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/desivideo.py b/cyberdrop_dl/crawlers/desivideo.py
index a221f2224..d488ec157 100644
--- a/cyberdrop_dl/crawlers/desivideo.py
+++ b/cyberdrop_dl/crawlers/desivideo.py
@@ -31,16 +31,16 @@ class DesiVideoCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["search"] if query := scrape_item.url.query.get("s"):
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case ["videos", video_id, *_]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case _:
raise ValueError
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
if await self.check_complete_from_referer(scrape_item.url):
- return None
+ return
soup = await self.request_soup(scrape_item.url)
video_url = self.parse_url(Selector.VIDEO_SRC(soup))
@@ -49,7 +49,7 @@ class DesiVideoCrawler(Crawler):
scrape_item.uploaded_at = json_ld.upload_date(soup)
custom_filename = self.create_custom_filename(title, ext, file_id=video_id)
- return await self.handle_file(video_url, scrape_item, video_url.name, ext, custom_filename=custom_filename)
+ await self.handle_file(video_url, scrape_item, video_url.name, ext, custom_filename=custom_filename)
@error_handling_wrapper
async def search(self, scrape_item: ScrapeItem, query: str) -> None:
diff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.py
index ac4a81cfd..eab3079f5 100644
--- a/cyberdrop_dl/crawlers/dirtyship.py
+++ b/cyberdrop_dl/crawlers/dirtyship.py
@@ -38,9 +38,9 @@ class DirtyShipCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["tag" | "category" as type_, _]:
- return await self.playlist(scrape_item, type_)
+ await self.playlist(scrape_item, type_)
case [_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.py
index ecf51c97b..54cbda1b4 100644
--- a/cyberdrop_dl/crawlers/doodstream.py
+++ b/cyberdrop_dl/crawlers/doodstream.py
@@ -55,7 +55,7 @@ class DoodStreamCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["e", _, *_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/dropbox.py b/cyberdrop_dl/crawlers/dropbox.py
index 49daa98ee..6eed85583 100644
--- a/cyberdrop_dl/crawlers/dropbox.py
+++ b/cyberdrop_dl/crawlers/dropbox.py
@@ -49,13 +49,13 @@ class DropboxCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["s" | "sh", _, *_]:
- return await self.follow_db_redirect(scrape_item)
+ await self.follow_db_redirect(scrape_item)
case ["scl", "fi", _, *_]:
- return await self.file(scrape_item)
+ await self.file(scrape_item)
case ["scl", "fo", link_key, secure_hash]:
- return await self.folder(scrape_item, link_key, secure_hash)
+ await self.folder(scrape_item, link_key, secure_hash)
case ["scl", "fo", link_key, secure_hash, _]:
- return await self.file(scrape_item)
+ await self.file(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/e621.py b/cyberdrop_dl/crawlers/e621.py
index 9faac140c..dccbb9145 100644
--- a/cyberdrop_dl/crawlers/e621.py
+++ b/cyberdrop_dl/crawlers/e621.py
@@ -31,11 +31,11 @@ class E621Crawler(Crawler):
match scrape_item.url.parts[1:]:
case ["posts"] if tags := scrape_item.url.query.get("tags"):
- return await self.tags(scrape_item, tags)
+ await self.tags(scrape_item, tags)
case ["posts", post_id]:
- return await self.post(scrape_item, post_id)
+ await self.post(scrape_item, post_id)
case ["pools", pool_id]:
- return await self.pool(scrape_item, pool_id)
+ await self.pool(scrape_item, pool_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/efukt.py b/cyberdrop_dl/crawlers/efukt.py
index 1cf3864c4..5b85ea965 100644
--- a/cyberdrop_dl/crawlers/efukt.py
+++ b/cyberdrop_dl/crawlers/efukt.py
@@ -47,19 +47,20 @@ class EfuktCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["pics", _]:
- return await self.media(scrape_item)
+ await self.media(scrape_item)
case ["view.gif.php"] if scrape_item.url.query.get("id"):
- return await self.media(scrape_item)
+ await self.media(scrape_item)
case ["series", _]:
- return await self.series(scrape_item)
+ await self.series(scrape_item)
case [slug]:
if slug.isdigit():
- return await self.homepage(scrape_item)
- if slug.endswith(".html"):
- return await self.media(scrape_item)
- raise ValueError
+ await self.homepage(scrape_item)
+ elif slug.endswith(".html"):
+ await self.media(scrape_item)
+ else:
+ raise ValueError
case []:
- return await self.homepage(scrape_item)
+ await self.homepage(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/ehentai.py b/cyberdrop_dl/crawlers/ehentai.py
index 9c3ffbce0..e4c0e1aad 100644
--- a/cyberdrop_dl/crawlers/ehentai.py
+++ b/cyberdrop_dl/crawlers/ehentai.py
@@ -38,9 +38,9 @@ class EHentaiCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["g", _, gallery_id]:
- return await self.gallery(scrape_item, gallery_id)
+ await self.gallery(scrape_item, gallery_id)
case ["s", _, _]:
- return await self.image(scrape_item)
+ await self.image(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/eightmuses.py b/cyberdrop_dl/crawlers/eightmuses.py
index 50817e67d..991637a3d 100644
--- a/cyberdrop_dl/crawlers/eightmuses.py
+++ b/cyberdrop_dl/crawlers/eightmuses.py
@@ -26,9 +26,9 @@ class EightMusesCrawler(Crawler):
FOLDER_DOMAIN: ClassVar[str] = "8Muses"
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if "album" in scrape_item.url.parts:
- return await self.album(scrape_item)
- raise ValueError
+ if "album" not in scrape_item.url.parts:
+ raise ValueError
+ await self.album(scrape_item)
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.py
index b6f75b149..f4c91d398 100644
--- a/cyberdrop_dl/crawlers/eporner.py
+++ b/cyberdrop_dl/crawlers/eporner.py
@@ -75,23 +75,23 @@ class EpornerCrawler(Crawler):
FOLDER_DOMAIN: ClassVar[str] = "ePorner"
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
- async def fetch(self, scrape_item: ScrapeItem) -> None: # noqa: PLR0911
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [slug, *_] if slug.startswith("video-"):
video_id = slug.removeprefix("video-")
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["hd-porn" | "embed", video_id, *_]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["cat" | "channel" | "search" | "pornstar" | "tag", *_]:
- return await self.playlist(scrape_item)
+ await self.playlist(scrape_item)
case ["gallery", *_]:
- return await self.gallery(scrape_item)
+ await self.gallery(scrape_item)
case ["profile", username, *_]:
- return await self.profile(scrape_item, username)
+ await self.profile(scrape_item, username)
case ["photo", photo_id, *_]:
- return await self.photo(scrape_item, photo_id)
+ await self.photo(scrape_item, photo_id)
case ["search-photos", query, *_]:
- return await self.search_photos(scrape_item, query)
+ await self.search_photos(scrape_item, query)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/erome.py b/cyberdrop_dl/crawlers/erome.py
index 5bf4cf211..8899fbc80 100644
--- a/cyberdrop_dl/crawlers/erome.py
+++ b/cyberdrop_dl/crawlers/erome.py
@@ -32,9 +32,9 @@ class EromeCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["a", album_id, *_]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case ["search", *_] if query := scrape_item.url.query.get("q"):
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case [name]:
await self.profile(scrape_item, name)
case _:
diff --git a/cyberdrop_dl/crawlers/fapello.py b/cyberdrop_dl/crawlers/fapello.py
index c187b869f..d14b28673 100644
--- a/cyberdrop_dl/crawlers/fapello.py
+++ b/cyberdrop_dl/crawlers/fapello.py
@@ -33,9 +33,9 @@ class FapelloComCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [model]:
- return await self.model(scrape_item, model)
+ await self.model(scrape_item, model)
case [model, _post_id]:
- return await self.post(scrape_item, model)
+ await self.post(scrape_item, model)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.py
index 36a731128..397bef95f 100644
--- a/cyberdrop_dl/crawlers/filester.py
+++ b/cyberdrop_dl/crawlers/filester.py
@@ -45,9 +45,9 @@ class FilesterCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["d", slug]:
- return await self.file(scrape_item, slug)
+ await self.file(scrape_item, slug)
case ["f", slug]:
- return await self.folder(scrape_item, slug)
+ await self.folder(scrape_item, slug)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/flickr.py b/cyberdrop_dl/crawlers/flickr.py
index 961e32fb9..46f91e896 100644
--- a/cyberdrop_dl/crawlers/flickr.py
+++ b/cyberdrop_dl/crawlers/flickr.py
@@ -28,10 +28,10 @@ class FlickrCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["photos", user, "albums", photoset_id, *_]:
- return await self.photoset(scrape_item, user, photoset_id)
+ await self.photoset(scrape_item, user, photoset_id)
case ["photos", user, photo_id, *_]:
scrape_item.url = self.PRIMARY_URL / "photos" / user / photo_id
- return await self.photo(scrape_item, photo_id)
+ await self.photo(scrape_item, photo_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/fourchan.py b/cyberdrop_dl/crawlers/fourchan.py
index a97094787..431c1565f 100644
--- a/cyberdrop_dl/crawlers/fourchan.py
+++ b/cyberdrop_dl/crawlers/fourchan.py
@@ -39,11 +39,11 @@ class FourChanCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [board, "thread", thread_id, *_]:
- return await self.thread(scrape_item, board, thread_id)
+ await self.thread(scrape_item, board, thread_id)
case [board]:
- return await self.board(scrape_item, board)
+ await self.board(scrape_item, board)
case [board, _]:
- return await self.board(scrape_item, board)
+ await self.board(scrape_item, board)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/fsiblog.py b/cyberdrop_dl/crawlers/fsiblog.py
index 6771c87d2..0b1d7249a 100644
--- a/cyberdrop_dl/crawlers/fsiblog.py
+++ b/cyberdrop_dl/crawlers/fsiblog.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, URLConfig, auto_task_id
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import json_ld, open_graph
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -17,16 +17,17 @@ class Selector:
ARTICLE = ".elementor-post__title a"
+@URLConfig(allow_empty_path=True)
class FSIBlogCrawler(Crawler):
- SUPPORTED_DOMAINS = "fsiblog5.com", "fsiblog5.club"
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "fsiblog5.com", "fsiblog5.club"
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Posts": "/<category>/<title>",
"Search": "?s=<query>",
}
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://fsiblog5.com")
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.fsiblogxx.com")
DOMAIN: ClassVar[str] = "fsiblog.com"
FOLDER_DOMAIN: ClassVar[str] = "FSIBlog"
- OLD_DOMAINS = (
+ OLD_DOMAINS: ClassVar[tuple[str, ...]] = (
"fsiblog.com",
"fsiblog1.com",
"fsiblog2.com",
@@ -37,16 +38,19 @@ class FSIBlogCrawler(Crawler):
"fsiblog2.club",
"fsiblog3.club",
"fsiblog4.club",
+ "fsiblog5.com",
)
DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} - {title}"
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if query := scrape_item.url.query.get("s"):
- return await self.search(scrape_item, query)
- if len(scrape_item.url.parts) == 3:
- return await self.post(scrape_item)
- raise ValueError
+ match scrape_item.url.parts[1:]:
+ case [_, _]:
+ await self.post(scrape_item)
+ case [] if query := scrape_item.url.query.get("s"):
+ await self.search(scrape_item, query)
+ case _:
+ raise ValueError
@property
def separate_posts(self) -> bool:
diff --git a/cyberdrop_dl/crawlers/giphy.py b/cyberdrop_dl/crawlers/giphy.py
index 6b2d4df30..0b44d0e4f 100644
--- a/cyberdrop_dl/crawlers/giphy.py
+++ b/cyberdrop_dl/crawlers/giphy.py
@@ -22,7 +22,7 @@ class GiPhyCrawler(Crawler):
match scrape_item.url.parts[1:]:
case ["gifs", slug]:
gif_id = slug.rpartition("-")[-1]
- return await self.gif(scrape_item, gif_id)
+ await self.gif(scrape_item, gif_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/girlsreleased.py b/cyberdrop_dl/crawlers/girlsreleased.py
index 42456d3ec..56f4b49a5 100644
--- a/cyberdrop_dl/crawlers/girlsreleased.py
+++ b/cyberdrop_dl/crawlers/girlsreleased.py
@@ -40,13 +40,13 @@ class GirlsReleasedCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["set", set_id]:
- return await self.set(scrape_item, set_id)
+ await self.set(scrape_item, set_id)
case ["site", domain]:
- return await self.site(scrape_item, domain)
+ await self.site(scrape_item, domain)
case ["site", domain, "model", model_id, model_name]:
- return await self.site(scrape_item, domain, model_id, model_name)
+ await self.site(scrape_item, domain, model_id, model_name)
case ["model", model_id, name]:
- return await self.model(scrape_item, model_id, name)
+ await self.model(scrape_item, model_id, name)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/google_photos.py b/cyberdrop_dl/crawlers/google_photos.py
index 4245effde..af077503e 100644
--- a/cyberdrop_dl/crawlers/google_photos.py
+++ b/cyberdrop_dl/crawlers/google_photos.py
@@ -50,13 +50,14 @@ class GooglePhotosCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.host == "photos.app.goo.gl":
- return await self.follow_redirect(scrape_item)
+ await self.follow_redirect(scrape_item)
+ return
match scrape_item.url.parts[1:]:
case ["album" | "share" as type_, album_id, "photo", photo_id, *_]:
- return await self.album(scrape_item, type_, album_id, photo_id)
+ await self.album(scrape_item, type_, album_id, photo_id)
case ["album" | "share" as type_, album_id]:
- return await self.album(scrape_item, type_, album_id)
+ await self.album(scrape_item, type_, album_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/goonbox.py b/cyberdrop_dl/crawlers/goonbox.py
index 2e2e9dbb0..48a5eeef6 100644
--- a/cyberdrop_dl/crawlers/goonbox.py
+++ b/cyberdrop_dl/crawlers/goonbox.py
@@ -69,7 +69,8 @@ class GoonBoxCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
if self.is_subdomain(scrape_item.url):
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
+ return
match scrape_item.url.parts[1:]:
case ["img", file_id]:
diff --git a/cyberdrop_dl/crawlers/gupload.py b/cyberdrop_dl/crawlers/gupload.py
index 7ef2bf837..1cb4795be 100644
--- a/cyberdrop_dl/crawlers/gupload.py
+++ b/cyberdrop_dl/crawlers/gupload.py
@@ -32,7 +32,7 @@ class GUploadCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["data", "e", video_id]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/hitomi_la.py b/cyberdrop_dl/crawlers/hitomi_la.py
index 851bfbd2c..7a95bd7b7 100644
--- a/cyberdrop_dl/crawlers/hitomi_la.py
+++ b/cyberdrop_dl/crawlers/hitomi_la.py
@@ -73,19 +73,16 @@ class HitomiLaCrawler(Crawler):
match scrape_item.url.parts[1:]:
case ["cg" | "doujinshi" | "galleries" | "gamecg" | "imageset" | "manga" | "reader" | "anime", slug]:
gallery_id = slug.split("-", 1)[-1].removesuffix(".html")
- return await self.gallery(scrape_item, gallery_id)
-
+ await self.gallery(scrape_item, gallery_id)
case ["artist", "character", "group", "series", "tag", "type" as type_, slug]:
name, lang = _parse_slug(slug)
- return await self.collection(scrape_item, name, lang, colletion_type=type_)
-
+ await self.collection(scrape_item, name, lang, colletion_type=type_)
case ["search.html"] if scrape_item.url.query:
- return await self.search(scrape_item, scrape_item.url.query_string)
-
+ await self.search(scrape_item, scrape_item.url.query_string)
case [slug]:
match _parse_slug(slug):
case ["index", language]:
- return await self.index(scrape_item, language)
+ await self.index(scrape_item, language)
case _:
raise ValueError
case _:
diff --git a/cyberdrop_dl/crawlers/hohoj_tv.py b/cyberdrop_dl/crawlers/hohoj_tv.py
index b45bfca56..85a7adcc3 100644
--- a/cyberdrop_dl/crawlers/hohoj_tv.py
+++ b/cyberdrop_dl/crawlers/hohoj_tv.py
@@ -26,7 +26,7 @@ class HohojTVCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["lang_en", "video"] if video_id := scrape_item.url.query.get("id"):
- return await self.video(scrape_item, int(video_id))
+ await self.video(scrape_item, int(video_id))
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/hotpic.py b/cyberdrop_dl/crawlers/hotpic.py
index 9081f4f3e..c373d0aaf 100644
--- a/cyberdrop_dl/crawlers/hotpic.py
+++ b/cyberdrop_dl/crawlers/hotpic.py
@@ -31,11 +31,11 @@ class HotPicCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["album", album_id]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case ["i", _]:
- return await self.file(scrape_item)
+ await self.file(scrape_item)
case ["uploads" | "reddit", _, *_]:
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.py
index e401dc0c3..d65988fdf 100644
--- a/cyberdrop_dl/crawlers/imagebam.py
+++ b/cyberdrop_dl/crawlers/imagebam.py
@@ -48,11 +48,11 @@ class ImageBamCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["gallery", _]:
- return await self.gallery(scrape_item)
+ await self.gallery(scrape_item)
case ["image", _]:
- return await self.image(scrape_item)
+ await self.image(scrape_item)
case ["view", _]:
- return await self.view(scrape_item)
+ await self.view(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/imagevenue.py b/cyberdrop_dl/crawlers/imagevenue.py
index 134820b0c..59e2e72b5 100644
--- a/cyberdrop_dl/crawlers/imagevenue.py
+++ b/cyberdrop_dl/crawlers/imagevenue.py
@@ -39,11 +39,11 @@ class ImageVenueCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["img.php"] if scrape_item.url.query.get("image"):
- return await self.follow_redirect(scrape_item)
+ await self.follow_redirect(scrape_item)
case [image_id] if image_id.startswith("ME"):
- return await self.image(scrape_item)
+ await self.image(scrape_item)
case ["view", "o"] if scrape_item.url.query.get("i") and scrape_item.url.query.get("h"):
- return await self.image(scrape_item)
+ await self.image(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.py
index 39ea8fa5a..20d01ce25 100644
--- a/cyberdrop_dl/crawlers/imgbb.py
+++ b/cyberdrop_dl/crawlers/imgbb.py
@@ -35,15 +35,17 @@ class ImgBBCrawler(CheveretoCrawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
username, _, rest = scrape_item.url.host.partition(".imgbb.")
if username and rest:
- return await self.profile(scrape_item)
+ await self.profile(scrape_item)
+ return
if scrape_item.url.host == IMAGES_CDN:
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
+ return
match scrape_item.url.parts[1:]:
case ["album", album_id]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case [_]:
- return await self.media(scrape_item)
+ await self.media(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.py
index 24530b7ec..74c86b7de 100644
--- a/cyberdrop_dl/crawlers/imgur.py
+++ b/cyberdrop_dl/crawlers/imgur.py
@@ -49,14 +49,15 @@ class ImgurCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.host == _IMAGE_CDN.host:
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
+ return
match scrape_item.url.parts[1:]:
case ["a", album_id]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case [slug]:
image_id = slug.partition(".")[0]
- return await self.image(scrape_item, image_id)
+ await self.image(scrape_item, image_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.py
index ac5bc4b0f..430d0a40d 100644
--- a/cyberdrop_dl/crawlers/imx_to.py
+++ b/cyberdrop_dl/crawlers/imx_to.py
@@ -39,9 +39,9 @@ class ImxToCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["g", gallery_id]:
- return await self.gallery(scrape_item, gallery_id)
+ await self.gallery(scrape_item, gallery_id)
case ["i", _]:
- return await self.image(scrape_item)
+ await self.image(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/koofr.py b/cyberdrop_dl/crawlers/koofr.py
index 684e49c04..3f8080353 100644
--- a/cyberdrop_dl/crawlers/koofr.py
+++ b/cyberdrop_dl/crawlers/koofr.py
@@ -41,11 +41,12 @@ class KooFrCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.host == _SHORT_LINK_CDN.host:
- return await self.follow_redirect(scrape_item)
+ await self.follow_redirect(scrape_item)
+ return
match scrape_item.url.parts[1:]:
case ["links", content_id]:
- return await self.share(scrape_item, content_id)
+ await self.share(scrape_item, content_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/leakedzone.py b/cyberdrop_dl/crawlers/leakedzone.py
index 5180b8542..9dd7e8609 100644
--- a/cyberdrop_dl/crawlers/leakedzone.py
+++ b/cyberdrop_dl/crawlers/leakedzone.py
@@ -65,9 +65,9 @@ class LeakedZoneCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [_, "video", video_id]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case [_]:
- return await self.model(scrape_item)
+ await self.model(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/livestreamfails.py b/cyberdrop_dl/crawlers/livestreamfails.py
index d07a4740d..5625be881 100644
--- a/cyberdrop_dl/crawlers/livestreamfails.py
+++ b/cyberdrop_dl/crawlers/livestreamfails.py
@@ -35,9 +35,9 @@ class LivestreamFailsCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["clip", clip_id]:
- return await self.clip(scrape_item, clip_id)
+ await self.clip(scrape_item, clip_id)
case ["streamer", streamer_id]:
- return await self.streamer(scrape_item, streamer_id)
+ await self.streamer(scrape_item, streamer_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/livid.py b/cyberdrop_dl/crawlers/livid.py
index 0ce02b67b..0454e90d0 100644
--- a/cyberdrop_dl/crawlers/livid.py
+++ b/cyberdrop_dl/crawlers/livid.py
@@ -30,7 +30,7 @@ class LividCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["embed" | "watch", video_id]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/luxuretv.py b/cyberdrop_dl/crawlers/luxuretv.py
index 59e3e6fe9..dab245917 100644
--- a/cyberdrop_dl/crawlers/luxuretv.py
+++ b/cyberdrop_dl/crawlers/luxuretv.py
@@ -35,10 +35,10 @@ class LuxureTVCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["searchgate", "videos", query, *_]:
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case ["videos", slug, *_]:
video_id = slug.split("-")[-1].split(".")[0]
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/masahub.py b/cyberdrop_dl/crawlers/masahub.py
index 451da261e..830bc2b5e 100644
--- a/cyberdrop_dl/crawlers/masahub.py
+++ b/cyberdrop_dl/crawlers/masahub.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css, json_ld
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -19,22 +19,34 @@ class Selector:
class MasahubCrawler(Crawler):
- SUPPORTED_DOMAINS = "masa49.com", "masahub.com", "masahub2.com", "masafun.net", "lol49.com", "vido99.com"
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (
+ "masa49.com",
+ "masahub.com",
+ "masahub2.com",
+ "masafun.net",
+ "lol49.com",
+ "vido99.com",
+ )
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Videos": "/title",
"Search": "?s=<query>",
}
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://masahub.com")
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://lalamasa.mobi/")
DOMAIN: ClassVar[str] = "masahub.com"
FOLDER_DOMAIN: ClassVar[str] = "Masahub"
+ OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("masahub.com",)
NEXT_PAGE_SELECTOR: ClassVar[str] = Selector.NEXT_PAGE
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if query := scrape_item.url.query.get("s"):
- return await self.search(scrape_item, query)
- if len(scrape_item.url.parts) >= 2:
- return await self.video(scrape_item)
- raise ValueError
+ match scrape_item.url.parts[1:]:
+ case ["search", query]:
+ await self.search(scrape_item, query)
+ case [] if query := scrape_item.url.query.get("s"):
+ await self.search(scrape_item, query)
+ case [_, *_]:
+ await self.video(scrape_item)
+ case _:
+ raise ValueError
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 731d8e4dc..8288b3da0 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -85,13 +85,14 @@ class MediaFireCrawler(Crawler):
and (quick_key := scrape_item.url.query_string)
and not ("&" in quick_key or "=" in quick_key)
):
- return await self.file(scrape_item, quick_key)
+ await self.file(scrape_item, quick_key)
+ return
match scrape_item.url.parts[1:]:
case ["folder", folder_key, *_]:
- return await self.folder(scrape_item, folder_key)
+ await self.folder(scrape_item, folder_key)
case ["file" | "file_premium", quick_key, *_]:
- return await self.file(scrape_item, quick_key)
+ await self.file(scrape_item, quick_key)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index f4d90e691..54f4674e0 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -80,7 +80,7 @@ class MegaNzCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
if not self._logged_in:
- return None
+ return
info = self.core.parse_url(scrape_item.url, check_key=False)
if not info.public_key and scrape_item.password:
@@ -90,10 +90,11 @@ class MegaNzCrawler(Crawler):
if not info.public_key:
self.raise_exc(scrape_item, PasswordProtectedError("Public key missing from URL"))
- return None
+ return
if not info.is_folder:
- return await self.file(scrape_item, info.public_handle, info.public_key)
+ await self.file(scrape_item, info.public_handle, info.public_key)
+ return
await self.folder(scrape_item, info.public_handle, info.public_key, info.selected_folder, info.selected_file)
diff --git a/cyberdrop_dl/crawlers/megacloud.py b/cyberdrop_dl/crawlers/megacloud.py
index 59476ed33..ee7ba3220 100644
--- a/cyberdrop_dl/crawlers/megacloud.py
+++ b/cyberdrop_dl/crawlers/megacloud.py
@@ -48,7 +48,7 @@ class MegaCloudCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["embed-2", "v3", _, _]:
- return await self.embed_v3(scrape_item)
+ await self.embed_v3(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.py
index ee8abac84..64d89c258 100644
--- a/cyberdrop_dl/crawlers/missav.py
+++ b/cyberdrop_dl/crawlers/missav.py
@@ -41,9 +41,9 @@ class MissAVCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [*_, collection_type, name] if collection_type in _COLLECTION_TYPES:
- return await self.collection(scrape_item, collection_type, name)
+ await self.collection(scrape_item, collection_type, name)
case [*_, video_id]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.py
index 3c736286c..3fae8dafc 100644
--- a/cyberdrop_dl/crawlers/mixdrop.py
+++ b/cyberdrop_dl/crawlers/mixdrop.py
@@ -32,7 +32,7 @@ class MixDropCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["f" | "e", file_id]:
- return await self.file(scrape_item, file_id)
+ await self.file(scrape_item, file_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/monstercat.py b/cyberdrop_dl/crawlers/monstercat.py
index 24f345a63..00f3fe400 100644
--- a/cyberdrop_dl/crawlers/monstercat.py
+++ b/cyberdrop_dl/crawlers/monstercat.py
@@ -26,7 +26,7 @@ class MonstercatCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["release", release_slug]:
- return await self.release(scrape_item, release_slug)
+ await self.release(scrape_item, release_slug)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/multporn.py b/cyberdrop_dl/crawlers/multporn.py
index fd9f55af1..269373a31 100644
--- a/cyberdrop_dl/crawlers/multporn.py
+++ b/cyberdrop_dl/crawlers/multporn.py
@@ -31,9 +31,9 @@ class MultPornCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["comics" | "hentai_manga" | "gay_porn_comics" | "humor", _]:
- return await self.comic(scrape_item)
+ await self.comic(scrape_item)
case ["video", _]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/mydesi.py b/cyberdrop_dl/crawlers/mydesi.py
index 79e30234c..d938bb784 100644
--- a/cyberdrop_dl/crawlers/mydesi.py
+++ b/cyberdrop_dl/crawlers/mydesi.py
@@ -35,10 +35,9 @@ class MyDesiCrawler(Crawler):
init_page = int(init_page)
case _:
init_page = 1
- return await self.search(scrape_item, query, init_page)
-
+ await self.search(scrape_item, query, init_page)
case [_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/naughty_machinima.py b/cyberdrop_dl/crawlers/naughty_machinima.py
index be14787ae..b08d721d1 100644
--- a/cyberdrop_dl/crawlers/naughty_machinima.py
+++ b/cyberdrop_dl/crawlers/naughty_machinima.py
@@ -28,9 +28,9 @@ class NaughtyMachinimaCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["video", video_id, *_]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["album", album_id, *_]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/nhentai.py b/cyberdrop_dl/crawlers/nhentai.py
index 681bd5015..428de1714 100644
--- a/cyberdrop_dl/crawlers/nhentai.py
+++ b/cyberdrop_dl/crawlers/nhentai.py
@@ -49,9 +49,9 @@ class NHentaiCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["favorites" | "tag" | "search" | "parody" | "group" | "character" | "artist" as type_, _]:
- return await self.collection(scrape_item, type_)
+ await self.collection(scrape_item, type_)
case ["g", _]:
- return await self.gallery(scrape_item)
+ await self.gallery(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/noodle_magazine.py b/cyberdrop_dl/crawlers/noodle_magazine.py
index e14ec09b1..8faa4178b 100644
--- a/cyberdrop_dl/crawlers/noodle_magazine.py
+++ b/cyberdrop_dl/crawlers/noodle_magazine.py
@@ -54,9 +54,9 @@ class NoodleMagazineCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["watch", _]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case ["video", query]:
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/nova.py b/cyberdrop_dl/crawlers/nova.py
index df2d9f905..55fddcb2f 100644
--- a/cyberdrop_dl/crawlers/nova.py
+++ b/cyberdrop_dl/crawlers/nova.py
@@ -35,7 +35,7 @@ class NovaStorageCrawler(PixelDrainCrawler):
raise ValueError
match scrape_item.url.parts[1:]:
case ["d", *path] if path:
- return await self.filesystem(scrape_item, "/".join(path))
+ await self.filesystem(scrape_item, "/".join(path))
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/nsfw_xxx.py b/cyberdrop_dl/crawlers/nsfw_xxx.py
index c00141f3c..67c139746 100644
--- a/cyberdrop_dl/crawlers/nsfw_xxx.py
+++ b/cyberdrop_dl/crawlers/nsfw_xxx.py
@@ -37,15 +37,15 @@ class NsfwXXXCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["post", post_id]:
- return await self.post(scrape_item, post_id)
+ await self.post(scrape_item, post_id)
case ["user", user]:
- return await self.user(scrape_item, user)
+ await self.user(scrape_item, user)
case ["r", subreddit]:
- return await self.subreddit(scrape_item, subreddit)
+ await self.subreddit(scrape_item, subreddit)
case ["category", name]:
- return await self.category(scrape_item, name)
+ await self.category(scrape_item, name)
case ["search"] if query := scrape_item.url.query.get("q"):
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/nudeleted.py b/cyberdrop_dl/crawlers/nudeleted.py
index 5f19ca3c0..ffe567e11 100644
--- a/cyberdrop_dl/crawlers/nudeleted.py
+++ b/cyberdrop_dl/crawlers/nudeleted.py
@@ -27,11 +27,11 @@ class NudeletedCrawler(KernelVideoSharingCrawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["videos", *_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case ["search", query, *_]:
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case ["tags" as type_, name, *_]:
- return await self.collection(scrape_item, name, type_)
+ await self.collection(scrape_item, name, type_)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.py
index 2bf6d3ef0..66b2ddb6d 100644
--- a/cyberdrop_dl/crawlers/odnoklassniki.py
+++ b/cyberdrop_dl/crawlers/odnoklassniki.py
@@ -61,9 +61,9 @@ class OdnoklassnikiCrawler(Crawler):
case ["video", id_]:
if id_.startswith("c"):
return await self.channel(scrape_item, id_)
- return await self.video(scrape_item, id_)
+ await self.video(scrape_item, id_)
case ["profile", _, channel] if channel.startswith("c"):
- return await self.channel(scrape_item, channel)
+ await self.channel(scrape_item, channel)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/omegascans.py b/cyberdrop_dl/crawlers/omegascans.py
index d90cad7c0..689a8aa39 100644
--- a/cyberdrop_dl/crawlers/omegascans.py
+++ b/cyberdrop_dl/crawlers/omegascans.py
@@ -38,9 +38,9 @@ class OmegaScansCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["series", series_slug]:
- return await self.series(scrape_item, series_slug)
+ await self.series(scrape_item, series_slug)
case ["series", series_slug, chapter_slug]:
- return await self.chapter(scrape_item, series_slug, chapter_slug)
+ await self.chapter(scrape_item, series_slug, chapter_slug)
case ["file", *_]:
await self.direct_file(scrape_item)
case _:
diff --git a/cyberdrop_dl/crawlers/onefichier.py b/cyberdrop_dl/crawlers/onefichier.py
index 599ef4da9..ce35fe7d5 100644
--- a/cyberdrop_dl/crawlers/onefichier.py
+++ b/cyberdrop_dl/crawlers/onefichier.py
@@ -57,7 +57,7 @@ class OneFichierCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [] | [""] if _get_file_id(scrape_item.url.query):
- return await self.file(scrape_item)
+ await self.file(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/onepace.py b/cyberdrop_dl/crawlers/onepace.py
index 1aececa2a..590b2e95c 100644
--- a/cyberdrop_dl/crawlers/onepace.py
+++ b/cyberdrop_dl/crawlers/onepace.py
@@ -53,7 +53,7 @@ class OnePaceCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [*_, "watch"]:
- return await self.all_episodes(scrape_item)
+ await self.all_episodes(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/owncloud.py b/cyberdrop_dl/crawlers/owncloud.py
index d36976455..8586931aa 100644
--- a/cyberdrop_dl/crawlers/owncloud.py
+++ b/cyberdrop_dl/crawlers/owncloud.py
@@ -19,7 +19,7 @@ class OwnCloudCrawler(Crawler, is_generic=True):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["s", folder_token]:
- return await self.public_share(scrape_item, folder_token)
+ await self.public_share(scrape_item, folder_token)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/pillowcase.py b/cyberdrop_dl/crawlers/pillowcase.py
index f8066809e..698b45c47 100644
--- a/cyberdrop_dl/crawlers/pillowcase.py
+++ b/cyberdrop_dl/crawlers/pillowcase.py
@@ -30,9 +30,9 @@ class PillowCaseCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["f", file_id]:
- return await self.file(scrape_item, file_id)
+ await self.file(scrape_item, file_id)
case ["api", "download" | "get" | "metadata", slug, *_]:
- return await self.file(scrape_item, file_id=slug.partition(".")[0])
+ await self.file(scrape_item, file_id=slug.partition(".")[0])
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/pimp_bunny.py b/cyberdrop_dl/crawlers/pimp_bunny.py
index 9a59174cf..b787c18d8 100644
--- a/cyberdrop_dl/crawlers/pimp_bunny.py
+++ b/cyberdrop_dl/crawlers/pimp_bunny.py
@@ -62,17 +62,17 @@ class PimpBunnyCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["videos", _, *_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case ["onlyfans-models", model_name, *_]:
- return await self.model(scrape_item, model_name)
+ await self.model(scrape_item, model_name)
case ["albums", "models", model_name, *_]:
- return await self.model_albums(scrape_item, model_name)
+ await self.model_albums(scrape_item, model_name)
case ["albums", name, *_]:
- return await self.album(scrape_item, name)
+ await self.album(scrape_item, name)
case ["categories" as type_, name, *_]:
- return await self.collection(scrape_item, type_, name)
+ await self.collection(scrape_item, type_, name)
case ["tags" as type_, name, *_]:
- return await self.collection(scrape_item, type_, name)
+ await self.collection(scrape_item, type_, name)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/pimpandhost.py b/cyberdrop_dl/crawlers/pimpandhost.py
index 4c1e761c3..421d9a309 100644
--- a/cyberdrop_dl/crawlers/pimpandhost.py
+++ b/cyberdrop_dl/crawlers/pimpandhost.py
@@ -32,7 +32,7 @@ class PimpAndHostCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["album", album_id, *_]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case _:
await self.image(scrape_item)
diff --git a/cyberdrop_dl/crawlers/pinterest.py b/cyberdrop_dl/crawlers/pinterest.py
index 8633416e5..4ee95161f 100644
--- a/cyberdrop_dl/crawlers/pinterest.py
+++ b/cyberdrop_dl/crawlers/pinterest.py
@@ -46,11 +46,11 @@ class PinterestCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["pin", pin_id]:
- return await self.pin(scrape_item, pin_id)
+ await self.pin(scrape_item, pin_id)
case [user, slug]:
- return await self.board(scrape_item, user, slug)
+ await self.board(scrape_item, user, slug)
case [user]:
- return await self.user(scrape_item, user)
+ await self.user(scrape_item, user)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.py
index 7d6386dbc..bd2e10f5c 100644
--- a/cyberdrop_dl/crawlers/pixeldrain.py
+++ b/cyberdrop_dl/crawlers/pixeldrain.py
@@ -78,7 +78,7 @@ class PixelDrainProxyCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["u", _]:
- return self.handle_external_links(scrape_item)
+ self.handle_external_links(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.py
index 27290f1e9..bee31f5ec 100644
--- a/cyberdrop_dl/crawlers/pixhost.py
+++ b/cyberdrop_dl/crawlers/pixhost.py
@@ -36,13 +36,13 @@ class PixHostCrawler(Crawler):
case ["thumbs", _, *_] if self.is_subdomain(scrape_item.url):
src = _thumbnail_to_src(scrape_item.url)
scrape_item.url = _thumbnail_to_web_url(scrape_item.url)
- return await self.direct_file(scrape_item, src)
+ await self.direct_file(scrape_item, src)
case ["gallery", gallery_id]:
- return await self.gallery(scrape_item, gallery_id)
+ await self.gallery(scrape_item, gallery_id)
case ["show", _, *_]:
- return await self.image(scrape_item)
+ await self.image(scrape_item)
case ["images", _, *_]:
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.py
index f61680cd3..169a47a56 100644
--- a/cyberdrop_dl/crawlers/pkmncards.py
+++ b/cyberdrop_dl/crawlers/pkmncards.py
@@ -92,11 +92,11 @@ class PkmncardsCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["card", _, *_]:
- return await self.card(scrape_item)
+ await self.card(scrape_item)
case ["set", slug, *_]:
- return await self.card_set(scrape_item, slug)
+ await self.card_set(scrape_item, slug)
case ["series", slug, *_]:
- return await self.series(scrape_item, slug)
+ await self.series(scrape_item, slug)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.py
index 52e943906..99f69cb5f 100644
--- a/cyberdrop_dl/crawlers/pmvhaven.py
+++ b/cyberdrop_dl/crawlers/pmvhaven.py
@@ -41,13 +41,13 @@ class PMVHavenCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["video", slug] if video_id := slug.rsplit("_", 1)[-1]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["search"] if query := scrape_item.url.query.get("q"):
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case ["users" | "profile", user_id]:
- return await self.profile(scrape_item, user_id)
+ await self.profile(scrape_item, user_id)
case ["playlists", playlist_id]:
- return await self.playlist(scrape_item, playlist_id)
+ await self.playlist(scrape_item, playlist_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.py
index 52f507f25..149e538db 100644
--- a/cyberdrop_dl/crawlers/porntrex.py
+++ b/cyberdrop_dl/crawlers/porntrex.py
@@ -55,11 +55,11 @@ class PorntrexCrawler(Crawler):
match scrape_item.url.parts[1:]:
case ["video", video_id, *_]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["tags" | "categories" | "models" | "playlists" | "search" | "members" as type_, _, *_]:
- return await self.collection(scrape_item, type_)
+ await self.collection(scrape_item, type_)
case ["albums", album_id, *_]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/pornzog.py b/cyberdrop_dl/crawlers/pornzog.py
index 98daf8a04..e2426170a 100644
--- a/cyberdrop_dl/crawlers/pornzog.py
+++ b/cyberdrop_dl/crawlers/pornzog.py
@@ -26,7 +26,7 @@ class PornZogCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["video", _, _]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/postimg.py b/cyberdrop_dl/crawlers/postimg.py
index f41314260..7aed0d258 100644
--- a/cyberdrop_dl/crawlers/postimg.py
+++ b/cyberdrop_dl/crawlers/postimg.py
@@ -29,7 +29,7 @@ class PostImgCrawler(Crawler):
match scrape_item.url.parts[1:]:
case ["gallery", album_id, *_]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case _:
await self.image(scrape_item)
diff --git a/cyberdrop_dl/crawlers/realbooru.py b/cyberdrop_dl/crawlers/realbooru.py
index 21098aa99..3172689fc 100644
--- a/cyberdrop_dl/crawlers/realbooru.py
+++ b/cyberdrop_dl/crawlers/realbooru.py
@@ -37,10 +37,11 @@ class RealBooruCrawler(Crawler):
case ["index.php"]:
query = scrape_item.url.query
if tags := query.get("tags"):
- return await self.tags(scrape_item, tags)
- if query.get("id"):
- return await self.file(scrape_item)
- raise ValueError
+ await self.tags(scrape_item, tags)
+ elif query.get("id"):
+ await self.file(scrape_item)
+ else:
+ raise ValueError
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.py
index f5fc0c941..e935eeebe 100644
--- a/cyberdrop_dl/crawlers/redgifs.py
+++ b/cyberdrop_dl/crawlers/redgifs.py
@@ -74,12 +74,12 @@ class RedGifsCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["users", user_name]:
- return await self.user(scrape_item, user_name.lower())
+ await self.user(scrape_item, user_name.lower())
case ["i" | "watch" | "ifr", gif_id]:
- return await self.gif(scrape_item, _id(gif_id))
+ await self.gif(scrape_item, _id(gif_id))
case [_, _] | [_] if self.is_subdomain(scrape_item.url):
scrape_item.url = _canonical_url(scrape_item.url.name)
- return await self.gif(scrape_item, scrape_item.url.name)
+ await self.gif(scrape_item, scrape_item.url.name)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/rootz.py b/cyberdrop_dl/crawlers/rootz.py
index 1aa9919ae..58285ed8c 100644
--- a/cyberdrop_dl/crawlers/rootz.py
+++ b/cyberdrop_dl/crawlers/rootz.py
@@ -32,7 +32,7 @@ class RootzCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["d" | "file", short_code]:
- return await self.file(scrape_item, short_code)
+ await self.file(scrape_item, short_code)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/rule34vault.py b/cyberdrop_dl/crawlers/rule34vault.py
index 214a98b42..ed8815d31 100644
--- a/cyberdrop_dl/crawlers/rule34vault.py
+++ b/cyberdrop_dl/crawlers/rule34vault.py
@@ -32,11 +32,11 @@ class Rule34VaultCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["post", post_id]:
- return await self.post(scrape_item, post_id)
+ await self.post(scrape_item, post_id)
case ["playlists", "view", playlist_id]:
- return await self.playlist(scrape_item, playlist_id)
+ await self.playlist(scrape_item, playlist_id)
case [tags]:
- return await self.tags(scrape_item, *tags.split(r"|"))
+ await self.tags(scrape_item, *tags.split(r"|"))
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.py
index 7c69800cd..4c07eef1f 100644
--- a/cyberdrop_dl/crawlers/rule34video.py
+++ b/cyberdrop_dl/crawlers/rule34video.py
@@ -44,11 +44,11 @@ class Rule34VideoCrawler(KernelVideoSharingCrawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["video" | "videos", _, *_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case ["search" as type_, query]:
- return await self.search(scrape_item, query, type_)
+ await self.search(scrape_item, query, type_)
case ["tags" | "categories" | "members" | "models" as type_, _]:
- return await self.search(scrape_item, None, type_)
+ await self.search(scrape_item, None, type_)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/rule34xxx.py b/cyberdrop_dl/crawlers/rule34xxx.py
index 0613ea896..d0f0049a3 100644
--- a/cyberdrop_dl/crawlers/rule34xxx.py
+++ b/cyberdrop_dl/crawlers/rule34xxx.py
@@ -35,10 +35,11 @@ class Rule34XXXCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
if tags := scrape_item.url.query.get("tags"):
- return await self.tag(scrape_item, tags)
- if scrape_item.url.query.get("id"):
- return await self.file(scrape_item)
- raise ValueError
+ await self.tag(scrape_item, tags)
+ elif scrape_item.url.query.get("id"):
+ await self.file(scrape_item)
+ else:
+ raise ValueError
@error_handling_wrapper
async def tag(self, scrape_item: ScrapeItem, tags: str) -> None:
diff --git a/cyberdrop_dl/crawlers/rutube.py b/cyberdrop_dl/crawlers/rutube.py
index bfe6b9840..76b5ecae3 100644
--- a/cyberdrop_dl/crawlers/rutube.py
+++ b/cyberdrop_dl/crawlers/rutube.py
@@ -33,9 +33,9 @@ class RutubeCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["video", video_id]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["play", "embed", video_id]:
- return await self.embed(scrape_item, video_id)
+ await self.embed(scrape_item, video_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/scrolller.py b/cyberdrop_dl/crawlers/scrolller.py
index c864297f6..fbb37fb9f 100644
--- a/cyberdrop_dl/crawlers/scrolller.py
+++ b/cyberdrop_dl/crawlers/scrolller.py
@@ -26,7 +26,7 @@ class ScrolllerCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["r", subreddit]:
- return await self.subreddit(scrape_item, subreddit)
+ await self.subreddit(scrape_item, subreddit)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/soundgasm.py b/cyberdrop_dl/crawlers/soundgasm.py
index 01cb24282..f3e86b21a 100644
--- a/cyberdrop_dl/crawlers/soundgasm.py
+++ b/cyberdrop_dl/crawlers/soundgasm.py
@@ -26,9 +26,9 @@ class SoundGasmCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["u", user, _]:
- return await self.audio(scrape_item, user)
+ await self.audio(scrape_item, user)
case ["u", user]:
- return await self.user(scrape_item)
+ await self.user(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.py
index 1d553a00b..acbc3c3bb 100644
--- a/cyberdrop_dl/crawlers/spankbang.py
+++ b/cyberdrop_dl/crawlers/spankbang.py
@@ -74,18 +74,19 @@ class SpankBangCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [playlist_id, "playlist", _, _page]:
- return await self.playlist(scrape_item, playlist_id)
+ await self.playlist(scrape_item, playlist_id)
case [video_id, "video" | "embed" | "play", *_]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["profile", user, "videos"]:
- return await self.profile(scrape_item, user)
+ await self.profile(scrape_item, user)
case ["s", query, *_]:
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case [id_, "playlist", _]:
playlist_id, _, video_id = id_.partition("-")
if video_id:
- return await self.video(scrape_item, video_id)
- return await self.playlist(scrape_item, playlist_id)
+ await self.video(scrape_item, video_id)
+ return
+ await self.playlist(scrape_item, playlist_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/streamtape.py b/cyberdrop_dl/crawlers/streamtape.py
index 711efa37e..0950444c9 100644
--- a/cyberdrop_dl/crawlers/streamtape.py
+++ b/cyberdrop_dl/crawlers/streamtape.py
@@ -30,7 +30,7 @@ class StreamtapeCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["e" | "v", video_id, *_]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/tabootube.py b/cyberdrop_dl/crawlers/tabootube.py
index ccdb26588..8806c55d7 100644
--- a/cyberdrop_dl/crawlers/tabootube.py
+++ b/cyberdrop_dl/crawlers/tabootube.py
@@ -20,7 +20,7 @@ class TabooTubeCrawler(KernelVideoSharingCrawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["video", *_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/thothub.py b/cyberdrop_dl/crawlers/thothub.py
index 3d0e2f7ad..6df42c7f1 100644
--- a/cyberdrop_dl/crawlers/thothub.py
+++ b/cyberdrop_dl/crawlers/thothub.py
@@ -33,15 +33,15 @@ class ThotHubCrawler(KernelVideoSharingCrawler, ensure_trailing_slash=True):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:-1]:
case ["albums", album_id, _]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case ["videos", _, _]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case ["categories" | "tags" as type_, _]:
- return await self.search(scrape_item, None, type_)
+ await self.search(scrape_item, None, type_)
case ["search" as type_, query]:
- return await self.search(scrape_item, query, type_)
+ await self.search(scrape_item, query, type_)
case ["get_image", _, *_]:
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/tiktok.py b/cyberdrop_dl/crawlers/tiktok.py
index f5bb06c9b..52e53998e 100644
--- a/cyberdrop_dl/crawlers/tiktok.py
+++ b/cyberdrop_dl/crawlers/tiktok.py
@@ -118,10 +118,11 @@ class TikTokCrawler(Crawler):
case [_, "video" | "photo" | "v" as type_, media_id]:
media_id = media_id.removesuffix(".html")
if type_ != "photo" and self.download_src_quality_videos:
- return await self.src_quality_media(scrape_item, media_id)
- return await self.media(scrape_item, media_id)
+ await self.src_quality_media(scrape_item, media_id)
+ return
+ await self.media(scrape_item, media_id)
case [profile] if profile.startswith("@"):
- return await self.profile(scrape_item, profile.removeprefix("@"))
+ await self.profile(scrape_item, profile.removeprefix("@"))
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/tnaflix.py b/cyberdrop_dl/crawlers/tnaflix.py
index 24d860664..f54767aa5 100644
--- a/cyberdrop_dl/crawlers/tnaflix.py
+++ b/cyberdrop_dl/crawlers/tnaflix.py
@@ -46,11 +46,11 @@ class TNAFlixCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [_, _, name] if name.startswith("video") and (video_id := name.removeprefix("video")):
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["search" as type_] if query := scrape_item.url.query.get("what"):
- return await self.collection(scrape_item, type_, query)
+ await self.collection(scrape_item, type_, query)
case ["channel" | "profile" as type_, _]:
- return await self.collection(scrape_item, type_)
+ await self.collection(scrape_item, type_)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.py
index d351b75ae..d107464a7 100644
--- a/cyberdrop_dl/crawlers/tokyomotion.py
+++ b/cyberdrop_dl/crawlers/tokyomotion.py
@@ -51,21 +51,21 @@ class TokioMotionCrawler(Crawler):
match scrape_item.url.parts[1:]:
case ["video", video_id, *_]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["photo", _]:
- return await self.photo(scrape_item)
+ await self.photo(scrape_item)
case ["album", album_id, *_]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case ["user", user, *_]:
title = self.create_title(f"{user} [user]")
scrape_item.setup_as_profile(title)
- return await self.profile(scrape_item)
+ await self.profile(scrape_item)
case ["search"] if (
(query := scrape_item.url.query.get("search_query"))
and (query_type := scrape_item.url.query.get("search_type"))
and query_type != "users"
):
- return await self.search(scrape_item, query, query_type)
+ await self.search(scrape_item, query, query_type)
case _:
raise ValueError
@@ -122,11 +122,11 @@ class TokioMotionCrawler(Crawler):
case ["favorite", "videos"]:
scrape_item.setup_as_album("favorite")
scrape_item.append_folders("videos")
- return await self.crawl_children(scrape_item, Selector.VIDEO)
+ await self.crawl_children(scrape_item, Selector.VIDEO)
case ["videos"]:
scrape_item.setup_as_album("videos")
- return await self.crawl_children(scrape_item, Selector.VIDEO)
+ await self.crawl_children(scrape_item, Selector.VIDEO)
case ["favorite", "photos"]:
scrape_item.setup_as_album("favorite")
@@ -141,7 +141,7 @@ class TokioMotionCrawler(Crawler):
case ["albums"]:
scrape_item.setup_as_album("albums")
- return await self.crawl_children(scrape_item, Selector.ALBUM)
+ await self.crawl_children(scrape_item, Selector.ALBUM)
case []:
for path in ("albums", "favorite/photos", "videos", "favorite/videos"):
diff --git a/cyberdrop_dl/crawlers/toonily.py b/cyberdrop_dl/crawlers/toonily.py
index 7bbfb1de7..5125ccfb5 100644
--- a/cyberdrop_dl/crawlers/toonily.py
+++ b/cyberdrop_dl/crawlers/toonily.py
@@ -27,9 +27,9 @@ class ToonilyCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["serie", _]:
- return await self.series(scrape_item)
+ await self.series(scrape_item)
case ["serie", _, chapter] if chapter.startswith("chapter-"):
- return await self.chapter(scrape_item)
+ await self.chapter(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/tranny_one.py b/cyberdrop_dl/crawlers/tranny_one.py
index 12f67cfdb..06c006b79 100644
--- a/cyberdrop_dl/crawlers/tranny_one.py
+++ b/cyberdrop_dl/crawlers/tranny_one.py
@@ -41,16 +41,15 @@ class TrannyOneCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["view", video_id]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["search", query]:
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case ["pornstars", model_id, _]:
- return await self.model(scrape_item, model_id)
+ await self.model(scrape_item, model_id)
case ["pics", "album", album_id]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case ["work", "orig", _, _, _]:
- return await self.direct_file(scrape_item)
-
+ await self.direct_file(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/trannygem.py b/cyberdrop_dl/crawlers/trannygem.py
index 2e0b7ce71..5553f24a6 100644
--- a/cyberdrop_dl/crawlers/trannygem.py
+++ b/cyberdrop_dl/crawlers/trannygem.py
@@ -14,6 +14,6 @@ class TrannyGemCrawler(KernelVideoSharingCrawler, ensure_trailing_slash=True):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["videos", *_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/transfer_it.py b/cyberdrop_dl/crawlers/transfer_it.py
index 160b9df1d..731dce706 100644
--- a/cyberdrop_dl/crawlers/transfer_it.py
+++ b/cyberdrop_dl/crawlers/transfer_it.py
@@ -31,7 +31,7 @@ class TransferItCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["t", transfer_id]:
- return await self.transfer(scrape_item, transfer_id)
+ await self.transfer(scrape_item, transfer_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/transflix.py b/cyberdrop_dl/crawlers/transflix.py
index a8f85e1d1..cd19156e0 100644
--- a/cyberdrop_dl/crawlers/transflix.py
+++ b/cyberdrop_dl/crawlers/transflix.py
@@ -36,16 +36,16 @@ class TransflixCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["video", slug] if video_id := slug.rsplit("-", 1)[-1]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["search"] if query := scrape_item.url.query.get("q"):
- return await self.search(scrape_item, query)
+ await self.search(scrape_item, query)
case _:
raise ValueError
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
if await self.check_complete_from_referer(scrape_item.url):
- return None
+ return
soup = await self.request_soup(scrape_item.url)
title = open_graph.title(soup)
@@ -55,7 +55,7 @@ class TransflixCrawler(Crawler):
scrape_item.uploaded_at = _timestamp_from_filename(link.name)
custom_filename = self.create_custom_filename(title, ext, file_id=video_id)
- return await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
+ await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
@error_handling_wrapper
async def search(self, scrape_item: ScrapeItem, query: str) -> None:
diff --git a/cyberdrop_dl/crawlers/twitch.py b/cyberdrop_dl/crawlers/twitch.py
index f4a010c91..8d98c2f42 100644
--- a/cyberdrop_dl/crawlers/twitch.py
+++ b/cyberdrop_dl/crawlers/twitch.py
@@ -48,11 +48,11 @@ class TwitchCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [_, "v", video_id]:
- return await self.vod(scrape_item, video_id)
+ await self.vod(scrape_item, video_id)
case ["video" | "videos", video_id]:
- return await self.vod(scrape_item, video_id)
+ await self.vod(scrape_item, video_id)
case ["collections", collection_id]:
- return await self.collection(scrape_item, collection_id)
+ await self.collection(scrape_item, collection_id)
case [*_, "clip", slug]:
await self.clip(scrape_item, slug)
case ["embed"] if slug := scrape_item.url.query.get("clip"):
@@ -61,11 +61,11 @@ class TwitchCrawler(Crawler):
await self.clip(scrape_item, slug)
case _:
if video_id := scrape_item.url.query.get("video"):
- return await self.vod(scrape_item, video_id)
- if slug := scrape_item.url.query.get("clip"):
- return await self.clip(scrape_item, slug)
-
- raise ValueError
+ await self.vod(scrape_item, video_id)
+ elif slug := scrape_item.url.query.get("clip"):
+ await self.clip(scrape_item, slug)
+ else:
+ raise ValueError
def __post_init__(self) -> None:
self.api: TwitchAPI = TwitchAPI.from_crawler(self)
diff --git a/cyberdrop_dl/crawlers/twpornstars.py b/cyberdrop_dl/crawlers/twpornstars.py
index cba4574aa..4c7987f86 100644
--- a/cyberdrop_dl/crawlers/twpornstars.py
+++ b/cyberdrop_dl/crawlers/twpornstars.py
@@ -47,11 +47,11 @@ class TwPornstarsCrawler(TwimgCrawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["p", _post_id]:
- return await self.media(scrape_item)
+ await self.media(scrape_item)
case ["hashtag", _slug]:
- return await self.collection(scrape_item)
+ await self.collection(scrape_item)
case [_]:
- return await self.collection(scrape_item)
+ await self.collection(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/upload_ee.py b/cyberdrop_dl/crawlers/upload_ee.py
index ea2ccf330..7c186f4fd 100644
--- a/cyberdrop_dl/crawlers/upload_ee.py
+++ b/cyberdrop_dl/crawlers/upload_ee.py
@@ -20,7 +20,7 @@ class UploadEECrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["files", _, *_]:
- return await self.file(scrape_item)
+ await self.file(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/vidara.py b/cyberdrop_dl/crawlers/vidara.py
index 88e52fb4d..291ac50ab 100644
--- a/cyberdrop_dl/crawlers/vidara.py
+++ b/cyberdrop_dl/crawlers/vidara.py
@@ -30,7 +30,7 @@ class VidaraCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["e", video_id]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/vipr_dot_im.py b/cyberdrop_dl/crawlers/vipr_dot_im.py
index 8ab2114db..8204183aa 100644
--- a/cyberdrop_dl/crawlers/vipr_dot_im.py
+++ b/cyberdrop_dl/crawlers/vipr_dot_im.py
@@ -25,7 +25,7 @@ class ViprImCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [_]:
- return await self.image(scrape_item)
+ await self.image(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/voe_sx.py b/cyberdrop_dl/crawlers/voe_sx.py
index 71aa29c90..d0cd3d646 100644
--- a/cyberdrop_dl/crawlers/voe_sx.py
+++ b/cyberdrop_dl/crawlers/voe_sx.py
@@ -94,7 +94,7 @@ class VoeSxCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["e", video_id] | [video_id, "download"] | [video_id]:
- return await self.embed(scrape_item, video_id)
+ await self.embed(scrape_item, video_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/vsco.py b/cyberdrop_dl/crawlers/vsco.py
index 5b8c29485..00194e6a9 100644
--- a/cyberdrop_dl/crawlers/vsco.py
+++ b/cyberdrop_dl/crawlers/vsco.py
@@ -34,9 +34,9 @@ class VSCOCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [user, "gallery"]:
- return await self.gallery(scrape_item, user)
+ await self.gallery(scrape_item, user)
case [user, "media" | "video" as type_, media_id]:
- return await self.media(scrape_item, user, type_, media_id)
+ await self.media(scrape_item, user, type_, media_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/wetransfer.py b/cyberdrop_dl/crawlers/wetransfer.py
index d85e43a2d..a23466885 100644
--- a/cyberdrop_dl/crawlers/wetransfer.py
+++ b/cyberdrop_dl/crawlers/wetransfer.py
@@ -30,13 +30,13 @@ class WeTransferCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [_] if scrape_item.url.host == "we.tl":
- return await self.follow_redirect(scrape_item)
+ await self.follow_redirect(scrape_item)
case ["downloads", file_id, security_hash]:
- return await self.file(scrape_item, file_id, security_hash)
+ await self.file(scrape_item, file_id, security_hash)
case ["downloads", file_id, recipient_id, security_hash]:
- return await self.file(scrape_item, file_id, security_hash, recipient_id)
+ await self.file(scrape_item, file_id, security_hash, recipient_id)
case [*_] if "download." in scrape_item.url.host:
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/whyp_it.py b/cyberdrop_dl/crawlers/whyp_it.py
index b3d56ac0b..386f8ae0d 100644
--- a/cyberdrop_dl/crawlers/whyp_it.py
+++ b/cyberdrop_dl/crawlers/whyp_it.py
@@ -30,11 +30,11 @@ class WhypItCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["tracks", slug] if track_id := slug.rpartition("-")[-1]:
- return await self.track(scrape_item, track_id)
+ await self.track(scrape_item, track_id)
case ["users", slug] if user_id := slug.rpartition("-")[-1]:
- return await self.user(scrape_item, user_id)
+ await self.user(scrape_item, user_id)
case ["collections", slug] if collection_id := slug.rpartition("-")[-1]:
- return await self.collection(scrape_item, collection_id)
+ await self.collection(scrape_item, collection_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/wikifeet.py b/cyberdrop_dl/crawlers/wikifeet.py
index 4695e37fd..c491eec5c 100644
--- a/cyberdrop_dl/crawlers/wikifeet.py
+++ b/cyberdrop_dl/crawlers/wikifeet.py
@@ -27,8 +27,9 @@ class WikiFeetCrawler(Crawler):
match scrape_item.url.parts[1:]:
case [name]:
if ".jpg" in name:
- return await self.direct_file(scrape_item)
- return await self.celeb(scrape_item, name)
+ await self.direct_file(scrape_item)
+ return
+ await self.celeb(scrape_item, name)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/xasiat.py b/cyberdrop_dl/crawlers/xasiat.py
index 1ccdb913a..098bdd526 100644
--- a/cyberdrop_dl/crawlers/xasiat.py
+++ b/cyberdrop_dl/crawlers/xasiat.py
@@ -24,10 +24,10 @@ class XasiatCrawler(KernelVideoSharingCrawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["albums", album_id, _, *_]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case ["videos", _, _, *_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case ["get_image", _, *_]:
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/xgroovy.py b/cyberdrop_dl/crawlers/xgroovy.py
index 28795eca8..9f666eafe 100644
--- a/cyberdrop_dl/crawlers/xgroovy.py
+++ b/cyberdrop_dl/crawlers/xgroovy.py
@@ -51,15 +51,15 @@ class XGroovyCrawler(FluidPlayerCrawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [*_, "videos" | "gifs", video_id, _]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case [*_, "pornstars" as type_, _]:
- return await self.collection(scrape_item, type_)
+ await self.collection(scrape_item, type_)
case [*_, "categories" | "channels" | "search" | "tag" as type_, slug]:
- return await self.collection(scrape_item, type_, slug)
+ await self.collection(scrape_item, type_, slug)
case [*_, "photos", album_id, _]:
- return await self.album(scrape_item, album_id)
+ await self.album(scrape_item, album_id)
case [*_, "contents", "albums", "sources", _, _, _]:
- return await self.direct_file(scrape_item)
+ await self.direct_file(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.py
index 52e606f2e..846f68dcb 100644
--- a/cyberdrop_dl/crawlers/xvideos.py
+++ b/cyberdrop_dl/crawlers/xvideos.py
@@ -90,13 +90,13 @@ class XVideosCrawler(Crawler):
raise ValueError
match scrape_item.url.parts[1:]:
case [part, _] if part.startswith("video"):
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case [_ as part, _] if part in _EXTENDED_ACCOUNTS:
- return await self.account(scrape_item)
+ await self.account(scrape_item)
case [_ as part, _, "photos" | "post", gallery_id, *_] if part in _EXTENDED_ACCOUNTS:
- return await self.gallery(scrape_item, gallery_id)
+ await self.gallery(scrape_item, gallery_id)
case [_ as part] if part not in _EXTENDED_ACCOUNTS: # channel
- return await self.account(scrape_item)
+ await self.account(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/xxxbunker.py b/cyberdrop_dl/crawlers/xxxbunker.py
index 567f0a27d..b8b2e88dc 100644
--- a/cyberdrop_dl/crawlers/xxxbunker.py
+++ b/cyberdrop_dl/crawlers/xxxbunker.py
@@ -43,9 +43,9 @@ class XXXBunkerCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["search" | "categories" as type_, name]:
- return await self.playlist(scrape_item, name, type_)
+ await self.playlist(scrape_item, name, type_)
case [username, "favoritevideos" as type_]:
- return await self.playlist(scrape_item, f"user {username}", type_)
+ await self.playlist(scrape_item, f"user {username}", type_)
case [_]:
await self.video(scrape_item)
case _:
diff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.py
index 6026d6110..5e2191e5d 100644
--- a/cyberdrop_dl/crawlers/yandex_disk.py
+++ b/cyberdrop_dl/crawlers/yandex_disk.py
@@ -81,12 +81,13 @@ class YandexDiskCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["d", folder_id, file_name]:
- return await self.folder(scrape_item, folder_id, file_name)
+ await self.folder(scrape_item, folder_id, file_name)
case ["d", folder_id]:
- return await self.folder(scrape_item, folder_id)
+ await self.folder(scrape_item, folder_id)
case ["i", _]:
- return await self.file(scrape_item)
- raise ValueError
+ await self.file(scrape_item)
+ case _:
+ raise ValueError
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/crawlers/youjizz.py b/cyberdrop_dl/crawlers/youjizz.py
index b5ffce73a..4d35de2dc 100644
--- a/cyberdrop_dl/crawlers/youjizz.py
+++ b/cyberdrop_dl/crawlers/youjizz.py
@@ -39,10 +39,10 @@ class YouJizzCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["videos", "embed", video_id]:
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case ["videos", video_name]:
video_id = video_name.rsplit("-", 1)[-1].removesuffix(".html")
- return await self.video(scrape_item, video_id)
+ await self.video(scrape_item, video_id)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/ytboob.py b/cyberdrop_dl/crawlers/ytboob.py
index 98a4f479d..3f65c605e 100644
--- a/cyberdrop_dl/crawlers/ytboob.py
+++ b/cyberdrop_dl/crawlers/ytboob.py
@@ -29,7 +29,7 @@ class YTboobCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [_]:
- return await self.video(scrape_item)
+ await self.video(scrape_item)
case _:
raise ValueError
diff --git a/cyberdrop_dl/crawlers/yurivan.py b/cyberdrop_dl/crawlers/yurivan.py
index f5ef2f2bf..3f936c437 100644
--- a/cyberdrop_dl/crawlers/yurivan.py
+++ b/cyberdrop_dl/crawlers/yurivan.py
@@ -32,9 +32,9 @@ class YuriVanCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["story", story_id, "read"] if chapter := scrape_item.url.query.get("chapter"):
- return await self.chapter(scrape_item, story_id, int(chapter))
+ await self.chapter(scrape_item, story_id, int(chapter))
case ["story", story_id, *_]:
- return await self.story(scrape_item, story_id)
+ await self.story(scrape_item, story_id)
case _:
raise ValueError