Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: add missing @error_handling_wrapper (#1921)

NTFSvolumed8ba654b78ea
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6b2e63e7a..700b2fb5d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Do not crash on HTTP errors from individual photos (Flickr)
+- Do not crash on HTTP errors from individual files (Google Drive)
+
### Security
## [9.12.0] - 2026-05-27
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index 627ad7e72..42a5a566b 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -184,27 +184,18 @@ class MessageBoardCrawler(Crawler, is_abc=True):
@abstractmethod
async def thread(self, scrape_item: ScrapeItem, /, thread: ThreadProtocol) -> None: ...
- async def forum(self, scrape_item: ScrapeItem) -> None:
- # Subclasses can define custom logic for this method.
- # They would need to also override `fetch` since the default fetch does not take this method into account
- raise NotImplementedError
-
async def resolve_confirmation_link(self, url: AbsoluteHttpURL, /) -> AbsoluteHttpURL | None:
# Not every forum has confirmation link so overriding this method is optional
# Implementation of this method MUST return `None` instead of raising an error
raise NotImplementedError
async def __async_post_init__(self) -> None:
- await self.login()
-
- @final
- async def login(self) -> None:
if self.login_required is None:
return
if not self._logged_in:
login_url = self.PRIMARY_URL / "login"
- await self._login(login_url)
+ await self.login(login_url)
@final
@property
@@ -228,22 +219,22 @@ class MessageBoardCrawler(Crawler, is_abc=True):
if self.is_attachment(scrape_item.url):
return await self.handle_internal_link(scrape_item)
if is_confirmation_link(scrape_item.url):
- return await self.follow_confirmation_link(scrape_item)
+ return await self._follow_confirmation_link(scrape_item)
- await self.fetch_thread(scrape_item)
+ await self._fetch_thread(scrape_item)
- async def fetch_thread(self, scrape_item: ScrapeItem) -> None:
+ async def _fetch_thread(self, scrape_item: ScrapeItem) -> None:
thread_part_index = len(self.PRIMARY_URL.parts)
# https://github.com/Cyberdrop-DL/cyberdrop-dl/issues/1165#issuecomment-3086739753
if self.PRIMARY_URL.parts[-1] == "":
thread_part_index -= 1
match scrape_item.url.parts[thread_part_index:]:
case [thread_part, thread_name_and_id, *_] if thread_part in self.THREAD_PART_NAMES:
- self.check_thread_recursion(scrape_item)
+ self._check_thread_recursion(scrape_item)
thread = self.parse_thread(scrape_item.url, thread_name_and_id)
return await self.thread(scrape_item, thread)
case ["goto" | "posts", _, *_]:
- self.check_thread_recursion(scrape_item)
+ self._check_thread_recursion(scrape_item)
return await self.follow_redirect(scrape_item)
case _:
raise ValueError
@@ -259,7 +250,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):
return by_parts or by_host
@final
- async def follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:
+ async def _follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:
url = await self.resolve_confirmation_link(scrape_item.url)
if url: # If there was an error, this will be None
scrape_item.url = url
@@ -267,7 +258,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):
return self.handle_external_links(scrape_item)
@final
- def check_thread_recursion(self, scrape_item: ScrapeItem) -> None:
+ def _check_thread_recursion(self, scrape_item: ScrapeItem) -> None:
if self.stop_thread_recursion(scrape_item):
parents = f"{len(scrape_item.parent_threads)} parent thread(s)"
msg = (
@@ -278,10 +269,10 @@ class MessageBoardCrawler(Crawler, is_abc=True):
)
raise MaxChildrenError(msg)
- self.limit_nexted_thread_folders(scrape_item)
+ self._limit_nexted_thread_folders(scrape_item)
@final
- def limit_nexted_thread_folders(self, scrape_item: ScrapeItem) -> None:
+ def _limit_nexted_thread_folders(self, scrape_item: ScrapeItem) -> None:
if self.max_thread_folder_depth is None:
return
n_parents = len(scrape_item.parent_threads)
@@ -325,7 +316,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):
await self.handle_file(link, new_scrape_item, filename, ext)
@final
- async def write_last_forum_post(self, thread_url: AbsoluteHttpURL, last_post_url: AbsoluteHttpURL | None) -> None:
+ async def _write_last_forum_post(self, thread_url: AbsoluteHttpURL, last_post_url: AbsoluteHttpURL | None) -> None:
if not last_post_url or last_post_url == thread_url:
return
self.manager.logs.write_last_post_log(last_post_url)
@@ -334,7 +325,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):
# TODO: Define an unified workflow for crawlers to perform and check login
@final
@error_handling_wrapper
- async def _login(self, login_url: AbsoluteHttpURL) -> None:
+ async def login(self, login_url: AbsoluteHttpURL) -> None:
session_cookie = self.get_cookie_value(self.LOGIN_USER_COOKIE_NAME)
msg = f"No cookies found for {self.FOLDER_DOMAIN}"
if not session_cookie and self.login_required:
@@ -422,9 +413,9 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
raise ScrapeError("User Error", msg)
self.scraped_threads.add(thread.url)
- await self.process_thread(scrape_item, thread)
+ await self._thread(scrape_item, thread)
- async def process_thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None:
+ async def _thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None:
title: str = ""
last_post_url = thread.url
async for soup in self.thread_pager(scrape_item):
@@ -436,13 +427,13 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
raise
scrape_item.add_to_parent_title(title)
- continue_scraping, last_post_url = self.process_thread_page(scrape_item, thread, soup)
+ continue_scraping, last_post_url = self._thread_page(scrape_item, thread, soup)
if not continue_scraping:
break
- await self.write_last_forum_post(thread.url, last_post_url)
+ await self._write_last_forum_post(thread.url, last_post_url)
- def process_thread_page(
+ def _thread_page(
self, scrape_item: ScrapeItem, thread: ThreadProtocol, soup: BeautifulSoup
) -> tuple[bool, AbsoluteHttpURL]:
continue_scraping = False
@@ -576,6 +567,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
new_link = self.parse_url(link_str)
return await self.get_absolute_link(new_link)
+ @error_handling_wrapper
async def handle_internal_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:
link = link or scrape_item.url
slug = link.name or link.parent.name
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.py
index 13f63c253..3a140038c 100644
--- a/cyberdrop_dl/crawlers/_kvs.py
+++ b/cyberdrop_dl/crawlers/_kvs.py
@@ -127,7 +127,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
title = self.create_title(title)
scrape_item.setup_as_album(title)
- await self.iter_videos(scrape_item)
+ await self._iter_videos(scrape_item)
@error_handling_wrapper
async def profile(self, scrape_item: ScrapeItem) -> None:
@@ -139,13 +139,13 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
scrape_item.setup_as_profile(title)
if soup.select_one(Selector.PUBLIC_VIDEOS):
- await self.iter_videos(scrape_item, "public_videos")
+ await self._iter_videos(scrape_item, "public_videos")
if soup.select_one(Selector.FAVOURITE_VIDEOS):
- await self.iter_videos(scrape_item, "favourite_videos")
+ await self._iter_videos(scrape_item, "favourite_videos")
if soup.select_one(Selector.PRIVATE_VIDEOS):
- await self.iter_videos(scrape_item, "private_videos")
+ await self._iter_videos(scrape_item, "private_videos")
- async def iter_videos(self, scrape_item: ScrapeItem, video_category: str = "") -> None:
+ async def _iter_videos(self, scrape_item: ScrapeItem, video_category: str = "") -> None:
url = scrape_item.url / video_category if video_category else scrape_item.url
async for soup in self.web_pager(url):
for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS):
diff --git a/cyberdrop_dl/crawlers/cloudflare_stream.py b/cyberdrop_dl/crawlers/cloudflare_stream.py
index 038e78974..4077a8c5d 100644
--- a/cyberdrop_dl/crawlers/cloudflare_stream.py
+++ b/cyberdrop_dl/crawlers/cloudflare_stream.py
@@ -43,13 +43,12 @@ class CloudflareStreamCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["embed", _] if video_id := scrape_item.url.query.get("video"):
- return await self.video(scrape_item, video_id)
+ pass
case [video_id, *_]:
- return await self.video(scrape_item, video_id)
+ pass
case _:
raise ValueError
- async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
# https://developers.cloudflare.com/stream/viewing-videos/securing-your-stream/
try:
jwt = JSONWebToken.decode(video_id)
@@ -63,10 +62,10 @@ class CloudflareStreamCrawler(Crawler):
return None
scrape_item.url = _DEFAULT_VIDEO_CDN / video_id
- return await self._video(scrape_item, video_id, token)
+ return await self.video(scrape_item, video_id, token)
@error_handling_wrapper
- async def _video(self, scrape_item: ScrapeItem, video_id: str, token: str | None) -> None:
+ async def video(self, scrape_item: ScrapeItem, video_id: str, token: str | None) -> None:
if await self.check_complete_from_referer(scrape_item):
return
diff --git a/cyberdrop_dl/crawlers/discourse/__init__.py b/cyberdrop_dl/crawlers/discourse/__init__.py
index ef84a1b38..132c82d36 100644
--- a/cyberdrop_dl/crawlers/discourse/__init__.py
+++ b/cyberdrop_dl/crawlers/discourse/__init__.py
@@ -49,6 +49,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
def is_attachment(cls, link: AbsoluteHttpURL | str) -> bool:
return "uploads" in AbsoluteHttpURL(link).parts
+ @error_handling_wrapper
async def handle_internal_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:
link = link or scrape_item.url
if len(link.parts) < 5 and not link.suffix:
@@ -92,6 +93,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
def parse_thread(cls, url: AbsoluteHttpURL, thread_name_and_id: str) -> None: # pyright: ignore[reportIncompatibleMethodOverride]
raise NotImplementedError
+ @error_handling_wrapper
async def thread(self, scrape_item: ScrapeItem, /, thread: Topic) -> None: # pyright: ignore[reportIncompatibleMethodOverride]
await self.topic(scrape_item, thread)
@@ -111,7 +113,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
if last_post_id:
topic_url = self.PRIMARY_URL / topic.path.removeprefix("/")
post_url = topic_url / str(last_post_id)
- await self.write_last_forum_post(topic_url, post_url)
+ await self._write_last_forum_post(topic_url, post_url)
async def iter_posts(self, topic: Topic) -> AsyncIterable[AvailablePost]:
for offset in itertools.count(topic.init_post_number - 1, _MAX_POSTS_PER_REQUEST):
@@ -124,6 +126,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
if topic.init_post_number != 1 and self.scrape_single_forum_post:
return
+ @error_handling_wrapper
async def post(self, scrape_item: ScrapeItem, /, post: AvailablePost) -> None: # pyright: ignore[reportIncompatibleMethodOverride]
title = self.create_separate_post_title(post.title, str(post.id), post.created_at)
scrape_item.setup_as_post(title)
diff --git a/cyberdrop_dl/crawlers/flickr.py b/cyberdrop_dl/crawlers/flickr.py
index a96a36443..18ee94ec4 100644
--- a/cyberdrop_dl/crawlers/flickr.py
+++ b/cyberdrop_dl/crawlers/flickr.py
@@ -58,6 +58,7 @@ class FlickrCrawler(Crawler):
self.create_task(self._photo(new_scrape_item, photo))
scrape_item.add_children()
+ @error_handling_wrapper
async def photo(self, scrape_item: ScrapeItem, photo_id: str) -> None:
if await self.check_complete_from_referer(scrape_item):
return
diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.py
index 9f44747b2..c28b14bdf 100644
--- a/cyberdrop_dl/crawlers/google_drive.py
+++ b/cyberdrop_dl/crawlers/google_drive.py
@@ -130,6 +130,7 @@ class GoogleDriveCrawler(Crawler):
if index % 200 == 0:
await asyncio.sleep(0)
+ @error_handling_wrapper
async def file(self, scrape_item: ScrapeItem, file_id: str = "", doc: str | None = None) -> None:
version = int(file_id[0])
if version not in _KNOWN_FILE_ID_VERSIONS:
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index fdc5fbbb3..667bce2df 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -64,7 +64,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
def password(self) -> str | None:
return self.manager.config.auth.meganz.password or None
- async def __async_post_init__(self) -> None:
+ def __post_init__(self) -> None:
self._decryption_keys: dict[AbsoluteHttpURL, tuple[Crypto, int]] = {}
api = MegaAPI(self.manager.http_client._session)
api.user_agent = CDL_USER_AGENT
@@ -72,7 +72,9 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
speed_limiter = self.downloader.client.speed_limiter
self.downloader = MegaDownloader(self.manager, self.DOMAIN) # pyright: ignore[reportIncompatibleVariableOverride]
self.downloader.client.speed_limiter = speed_limiter
- await self.login()
+
+ async def __async_post_init__(self) -> None:
+ await self._login()
async def fetch(self, scrape_item: ScrapeItem) -> None:
if not self._logged_in:
@@ -163,7 +165,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
media_item.extra_info.setdefault(self.DOMAIN, {})["key"] = self._decryption_keys.pop(media_item.url)
await super().handle_media_item(media_item, m3u8)
- async def login(self) -> None:
+ async def _login(self) -> None:
# This takes a really long time (dozens of seconds)
# TODO: Add a way to cache this login
# TODO: Show some logging message / UI about login
diff --git a/cyberdrop_dl/crawlers/nsfw_xxx.py b/cyberdrop_dl/crawlers/nsfw_xxx.py
index 29e96bebe..5b1fe63ac 100644
--- a/cyberdrop_dl/crawlers/nsfw_xxx.py
+++ b/cyberdrop_dl/crawlers/nsfw_xxx.py
@@ -48,23 +48,26 @@ class NsfwXXXCrawler(Crawler):
case _:
raise ValueError
+ @error_handling_wrapper
async def subreddit(self, scrape_item: ScrapeItem, subreddit: str) -> None:
api_url = self.PRIMARY_URL / "api/v1/source/r" / subreddit
await self._collection(scrape_item, api_url)
+ @error_handling_wrapper
async def category(self, scrape_item: ScrapeItem, name: str) -> None:
api_url = self.PRIMARY_URL / "api/v1/category" / name
await self._collection(scrape_item, api_url)
+ @error_handling_wrapper
async def search(self, scrape_item: ScrapeItem, query: str) -> None:
api_url = (self.PRIMARY_URL / "api/v1/search").with_query(q=query)
await self._collection(scrape_item, api_url, query)
+ @error_handling_wrapper
async def user(self, scrape_item: ScrapeItem, username: str) -> None:
api_url = self.PRIMARY_URL / "api/v1/user" / username
await self._collection(scrape_item, api_url, f"@{username}")
- @error_handling_wrapper
async def _collection(self, scrape_item: ScrapeItem, api_url: AbsoluteHttpURL, name: str | None = None) -> None:
title: str = ""
type_ = api_url.parts[3]
diff --git a/cyberdrop_dl/crawlers/pcloud.py b/cyberdrop_dl/crawlers/pcloud.py
index bfe83c78e..c07a87d42 100644
--- a/cyberdrop_dl/crawlers/pcloud.py
+++ b/cyberdrop_dl/crawlers/pcloud.py
@@ -88,7 +88,7 @@ class PCloudCrawler(Crawler):
self._iter_nodes(scrape_item, node.contents)
return None
- return await self.file(scrape_item, cast("File", node))
+ return await self._file(scrape_item, cast("File", node))
def _iter_nodes(self, scrape_item: ScrapeItem, nodes: Sequence[Node], *parents: str) -> None:
folders: list[Node] = []
@@ -109,7 +109,7 @@ class PCloudCrawler(Crawler):
for folder in folders:
self._iter_nodes(scrape_item, folder.contents, *parents, folder.name)
- async def file(self, scrape_item: ScrapeItem, file: File) -> None:
+ async def _file(self, scrape_item: ScrapeItem, file: File) -> None:
# https://docs.pcloud.com/methods/public_links/getpublinkdownload.html
link = await self._request_download_url(scrape_item, file)
@@ -120,7 +120,7 @@ class PCloudCrawler(Crawler):
db_url = (scrape_item.url.origin() / "file" / file._id).with_query(code=scrape_item.url.query["code"])
await self.handle_file(db_url, scrape_item, file.name, ext, debrid_link=link, custom_filename=filename)
- _file_task = auto_task_id(error_handling_wrapper(file))
+ _file_task = auto_task_id(error_handling_wrapper(_file))
async def _request_download_url(self, scrape_item: ScrapeItem, file: File) -> AbsoluteHttpURL:
path = "getmediatranscodepublink" if "video" in file.contenttype else "getpublinkdownload"
diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.py
index 83d09948b..ef228b567 100644
--- a/cyberdrop_dl/crawlers/pkmncards.py
+++ b/cyberdrop_dl/crawlers/pkmncards.py
@@ -127,7 +127,7 @@ class PkmncardsCrawler(Crawler):
download_url = self.parse_url(link_str)
simple_card = create_simple_card(title, download_url)
new_scrape_item = scrape_item.create_child(card_page_url)
- self.create_task(self.handle_simple_card(new_scrape_item, simple_card))
+ self.create_task(self._simple_card(new_scrape_item, simple_card))
scrape_item.add_children()
@error_handling_wrapper
@@ -155,7 +155,7 @@ class PkmncardsCrawler(Crawler):
custom_filename = self.create_custom_filename(card.full_name, ext)
await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
- async def handle_simple_card(self, scrape_item: ScrapeItem, simple_card: SimpleCard) -> None:
+ async def _simple_card(self, scrape_item: ScrapeItem, simple_card: SimpleCard) -> None:
@error_handling_wrapper
async def get_card_set(self, scrape_item: ScrapeItem) -> CardSet:
soup = await self.request_soup(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/pornpics.py b/cyberdrop_dl/crawlers/pornpics.py
index 5c5758f41..54cc15f3f 100644
--- a/cyberdrop_dl/crawlers/pornpics.py
+++ b/cyberdrop_dl/crawlers/pornpics.py
@@ -88,6 +88,7 @@ class PornPicsCrawler(Crawler):
filename, ext = self.get_filename_and_ext(new_scrape_item.url.name)
await self.handle_file(new_scrape_item.url, new_scrape_item, filename, ext)
+ @error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
scrape_item.album_id = scrape_item.url.parts[-2]
await self.direct_file(scrape_item)
diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.py
index 689f1c8e7..b3828599a 100644
--- a/cyberdrop_dl/crawlers/porntrex.py
+++ b/cyberdrop_dl/crawlers/porntrex.py
@@ -126,11 +126,11 @@ class PorntrexCrawler(Crawler):
for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS_OR_ALBUMS):
self.create_task(self.run(new_scrape_item))
- await self.proccess_additional_pages(scrape_item, last_page)
+ await self._ajax_pagination(scrape_item, last_page)
if "models" in scrape_item.url.parts:
# Additional album pages
- await self.proccess_additional_pages(scrape_item, last_page, block_id="list_albums_common_albums_list")
+ await self._ajax_pagination(scrape_item, last_page, block_id="list_albums_common_albums_list")
elif "members" in scrape_item.url.parts:
albums_url = scrape_item.url / "albums"
@@ -139,7 +139,7 @@ class PorntrexCrawler(Crawler):
for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ALBUMS, new_title_part="albums"):
self.create_task(self.run(new_scrape_item))
- async def proccess_additional_pages(self, scrape_item: ScrapeItem, last_page: int, **kwargs: str) -> None: # noqa: C901
+ async def _ajax_pagination(self, scrape_item: ScrapeItem, last_page: int, **kwargs: str) -> None: # noqa: C901
if last_page == 1:
return
block_id: str = "list_videos_common_videos_list_norm"
diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.py
index e3ca5e32f..dc1eaf79e 100644
--- a/cyberdrop_dl/crawlers/sendvid.py
+++ b/cyberdrop_dl/crawlers/sendvid.py
@@ -27,7 +27,7 @@ class SendVidCrawler(Crawler):
FOLDER_DOMAIN: ClassVar[str] = "SendVid"
async def fetch(self, scrape_item: ScrapeItem) -> None:
- scrape_item.url = self.get_streaming_url(scrape_item.url)
+ scrape_item.url = _get_streaming_url(scrape_item.url)
await self.video(scrape_item)
@error_handling_wrapper
@@ -43,11 +43,9 @@ class SendVidCrawler(Crawler):
except AssertionError:
raise ScrapeError(422, "Couldn't find video source") from None
link = self.parse_url(link_str)
- await self.handle_direct_link(scrape_item, link, title)
+ await self._video(scrape_item, link, title)
- async def handle_direct_link(
- self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None, title: str = ""
- ) -> None:
+ async def _video(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None, title: str = "") -> None:
link = link or scrape_item.url
canonical_url = link.with_query(None)
@@ -61,14 +59,15 @@ class SendVidCrawler(Crawler):
canonical_url, scrape_item, filename, ext, debrid_link=link, custom_filename=custom_filename
)
- def get_streaming_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- if is_cdn(url):
- video_id = url.name.split(".", 1)[0]
- return PRIMARY_URL.with_path(video_id)
- if "embed" in url.parts:
- return url.with_path(url.path.replace("/embed/", "/"), keep_query=True, keep_fragment=True)
- return url
+def _get_streaming_url(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ if is_cdn(url):
+ video_id = url.name.split(".", 1)[0]
+ return PRIMARY_URL.with_path(video_id)
+
+ if "embed" in url.parts:
+ return url.with_path(url.path.replace("/embed/", "/"), keep_query=True, keep_fragment=True)
+ return url
def is_cdn(url: AbsoluteHttpURL) -> bool:
diff --git a/cyberdrop_dl/crawlers/sex_dot_com.py b/cyberdrop_dl/crawlers/sex_dot_com.py
index 856f23698..bdbb4db90 100644
--- a/cyberdrop_dl/crawlers/sex_dot_com.py
+++ b/cyberdrop_dl/crawlers/sex_dot_com.py
@@ -55,7 +55,7 @@ class SexDotComCrawler(Crawler):
return (await self.request_json(data_url))["media"]
@error_handling_wrapper
- async def handle_media(self, scrape_item: ScrapeItem, item: dict[str, Any] | None) -> None:
+ async def _media(self, scrape_item: ScrapeItem, item: dict[str, Any] | None) -> None:
real_item = item or await self.get_media(scrape_item)
relative_url = real_item["relativeUrl"]
canonical_url = _SHORTS_URL / relative_url
@@ -82,10 +82,11 @@ class SexDotComCrawler(Crawler):
async def profile(self, scrape_item: ScrapeItem) -> None:
async for json_data in self.shorts_profile_paginator(scrape_item):
for item in json_data["page"]["items"]:
- await self.handle_media(scrape_item, item["media"])
+ await self._media(scrape_item, item["media"])
+ @error_handling_wrapper
async def post(self, scrape_item: ScrapeItem) -> None:
username = scrape_item.url.parts[2]
title = self.create_title(username)
scrape_item.setup_as_album(title)
- await self.handle_media(scrape_item, None)
+ await self._media(scrape_item, None)
diff --git a/cyberdrop_dl/crawlers/twitter_images.py b/cyberdrop_dl/crawlers/twitter_images.py
index d629b4518..0f4fd23ea 100644
--- a/cyberdrop_dl/crawlers/twitter_images.py
+++ b/cyberdrop_dl/crawlers/twitter_images.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem
+from cyberdrop_dl.utils import error_handling_wrapper
if TYPE_CHECKING:
from collections.abc import Generator
@@ -28,6 +29,7 @@ class TwimgCrawler(Crawler):
return await self.direct_file(scrape_item)
await self.photo(scrape_item)
+ @error_handling_wrapper
async def photo(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:
# https://developer.x.com/en/docs/x-api/v1/data-dictionary/object-model/entities#photo_format
link = url or scrape_item.url
diff --git a/cyberdrop_dl/crawlers/vbulletin/__init__.py b/cyberdrop_dl/crawlers/vbulletin/__init__.py
index b27e65d9e..678497c28 100644
--- a/cyberdrop_dl/crawlers/vbulletin/__init__.py
+++ b/cyberdrop_dl/crawlers/vbulletin/__init__.py
@@ -9,6 +9,7 @@ from typing_extensions import override
from cyberdrop_dl.crawlers.xenforo.xenforo import XenforoCrawler
from cyberdrop_dl.exceptions import MaxChildrenError, ScrapeError
+from cyberdrop_dl.utils import error_handling_wrapper
if TYPE_CHECKING:
import datetime
@@ -70,7 +71,7 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801
async def __async_post_init__(self) -> None:
if not self._logged_in:
login_url = self.PRIMARY_URL / "login.php"
- await self._login(login_url)
+ await self.login(login_url)
@override
async def check_login_with_request(self, *_: object, **_kwargs: object) -> tuple[str, bool]:
@@ -81,25 +82,26 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801
# TODO: Handle more URLs
if self.login_required and not self._logged_in:
return
- await self.fetch_thread(scrape_item)
+ await self._fetch_thread(scrape_item)
- async def process_thread(self, scrape_item: ScrapeItem, thread: Thread) -> None: # pyright: ignore[reportIncompatibleMethodOverride]
+ @override
+ async def _thread(self, scrape_item: ScrapeItem, thread: Thread) -> None: # pyright: ignore[reportIncompatibleMethodOverride]
title: str = ""
if thread.post_id and self.scrape_single_forum_post:
api_url = self.VBULLETIN_API_ENDPOINT.with_query({self.VBULLETIN_POST_QUERY_PARAM: str(thread.post_id)})
else:
api_url = self.VBULLETIN_API_ENDPOINT.with_query({self.VBULLETIN_THREAD_QUERY_PARAM: str(thread.id)})
- root_xml = await self.get_xml(api_url)
+ root_xml = _parse_xml(await self.request_text(api_url))
if (thread_element := root_xml.find("thread")) is not None:
title = self.create_title(thread_element.attrib["title"], thread_id=thread.id)
scrape_item.setup_as_forum(title)
else:
raise ScrapeError(422)
- await self.process_posts(scrape_item, thread, root_xml)
+ await self._posts(scrape_item, thread, root_xml)
- async def process_posts(self, scrape_item: ScrapeItem, thread: Thread, root_xml: ET.Element[str]) -> None:
+ async def _posts(self, scrape_item: ScrapeItem, thread: Thread, root_xml: ET.Element[str]) -> None:
if thread.page:
posts = itertools.islice(root_xml.iter("post"), (thread.page - 1) * _N_POSTS_PER_PAGE)
else:
@@ -122,8 +124,10 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801
if last_post_id:
last_post_url = thread.url.update_query({self.VBULLETIN_POST_QUERY_PARAM: str(last_post_id)})
- await self.write_last_forum_post(thread.url, last_post_url)
+ await self._write_last_forum_post(thread.url, last_post_url)
+ @override
+ @error_handling_wrapper
async def post(self, scrape_item: ScrapeItem, post: Post) -> None: # pyright: ignore[reportIncompatibleMethodOverride]
title = self.create_separate_post_title(post.title, str(post.id), None)
scrape_item.setup_as_post(title)
@@ -132,10 +136,11 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801
self.handle_external_links(new_scrap_item)
scrape_item.add_children()
- async def get_xml(self, url: AbsoluteHttpURL) -> ET.Element[str]:
- root_xml = ET.XML(await self.request_text(url))
- if error := root_xml.find("error"):
- details = error.attrib["details"]
- error_code = 403 if error.attrib["type"] == "permissions" and "unknown" not in details.casefold() else 422
- raise ScrapeError(error_code, message=details)
- return root_xml
+
+def _parse_xml(content: str) -> ET.Element[str]:
+ root_xml = ET.XML(content)
+ if error := root_xml.find("error"):
+ details = error.attrib["details"]
+ error_code = 403 if error.attrib["type"] == "permissions" and "unknown" not in details.casefold() else 422
+ raise ScrapeError(error_code, message=details)
+ return root_xml
diff --git a/cyberdrop_dl/crawlers/voe_sx.py b/cyberdrop_dl/crawlers/voe_sx.py
index 1f65f7473..8219a3fee 100644
--- a/cyberdrop_dl/crawlers/voe_sx.py
+++ b/cyberdrop_dl/crawlers/voe_sx.py
@@ -92,11 +92,7 @@ class VoeSxCrawler(Crawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
- case ["e", video_id]:
- return await self.embed(scrape_item, video_id)
- case [video_id, "download"]:
- return await self.embed(scrape_item, video_id)
- case [video_id]:
+ case ["e", video_id] | [video_id, "download"] | [video_id]:
return await self.embed(scrape_item, video_id)
case _:
raise ValueError
@@ -112,7 +108,7 @@ class VoeSxCrawler(Crawler):
text = await resp.text()
if match := _find_js_redirect(text):
scrape_item.url = self.parse_url(match.group(1), origin)
- self.create_task(self.redirect(scrape_item))
+ self.create_task(self._redirect(scrape_item))
return
soup = await resp.soup()
@@ -153,7 +149,7 @@ class VoeSxCrawler(Crawler):
self.handle_subs(scrape_item, custom_filename, video.subtitles)
- redirect = auto_task_id(fetch)
+ _redirect = auto_task_id(fetch)
def extract_voe_video(soup: BeautifulSoup, origin: AbsoluteHttpURL) -> VoeVideo:
diff --git a/cyberdrop_dl/crawlers/wordpress/__init__.py b/cyberdrop_dl/crawlers/wordpress/__init__.py
index 0c42332d4..5167fda99 100644
--- a/cyberdrop_dl/crawlers/wordpress/__init__.py
+++ b/cyberdrop_dl/crawlers/wordpress/__init__.py
@@ -79,7 +79,6 @@ class WordPressBaseCrawler(Crawler, is_abc=True):
def __init_subclass__(cls, **kwargs: Any) -> None:
super().__init_subclass__(**kwargs)
assert cls.fetch is WordPressBaseCrawler.fetch
- assert cls.fetch_with_date_range is WordPressBaseCrawler.fetch_with_date_range
@final
async def fetch(self, scrape_item: ScrapeItem) -> None:
@@ -87,7 +86,20 @@ class WordPressBaseCrawler(Crawler, is_abc=True):
scrape_item.url = scrape_item.url.with_query(None)
if date_range:
self.log.info(f"Scraping {scrape_item.url} with date range: {date_range.as_query()}")
- return await self.fetch_with_date_range(scrape_item, date_range)
+ match scrape_item.url.parts[1:]:
+ case ["posts"]:
+ return await self.all_posts(scrape_item, date_range)
+ case [ColletionType.CATEGORY.value, _]:
+ return await self.category_or_tag(scrape_item, ColletionType.CATEGORY, date_range)
+ case [ColletionType.TAG.value, _]:
+ return await self.category_or_tag(scrape_item, ColletionType.TAG, date_range)
+ case ["wp-json", *_]:
+ raise ValueError
+ case _:
+ if _match_date_from_path(scrape_item.url.parts[1:]):
+ # TODO: Handle this
+ raise ValueError
+ return await self.post(scrape_item)
@property
@final
@@ -100,23 +112,6 @@ class WordPressBaseCrawler(Crawler, is_abc=True):
def is_attachment(url: AbsoluteHttpURL) -> bool:
return "wp-content" in url.parts and bool(url.suffix)
- @final
- async def fetch_with_date_range(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None) -> None:
- match scrape_item.url.parts[1:]:
- case ["posts"]:
- return await self.all_posts(scrape_item, date_range)
- case [ColletionType.CATEGORY.value, _]:
- return await self.category_or_tag(scrape_item, ColletionType.CATEGORY, date_range)
- case [ColletionType.TAG.value, _]:
- return await self.category_or_tag(scrape_item, ColletionType.TAG, date_range)
- case ["wp-json", *_]:
- raise ValueError
-
- if _ := _match_date_from_path(scrape_item.url.parts[1:]):
- # TODO: Handle this
- raise ValueError
- return await self.post(scrape_item)
-
@abstractmethod
async def category_or_tag(
self, scrape_item: ScrapeItem, /, colletion_type: ColletionType, date_range: QueryDatetimeRange | None = None
@@ -129,18 +124,18 @@ class WordPressBaseCrawler(Crawler, is_abc=True):
async def all_posts(self, scrape_item: ScrapeItem, /, date_range: QueryDatetimeRange | None = None) -> None: ...
@final
- async def filter_post(
+ async def _filter_post(
self, scrape_item: ScrapeItem, post: Post, date_range: QueryDatetimeRange | None = None
) -> None:
if date_range and not date_range.is_in_range(post.date_gmt):
self.log.info(f"Skipping post {post.link} as it is out of date range. Post date: {[post.date_gmt]}")
return
new_scrape_item = scrape_item.create_child(self.parse_url(post.link))
- await self.handle_post(new_scrape_item, post)
+ await self._handle_post(new_scrape_item, post)
scrape_item.add_children()
@final
- async def handle_post(self, scrape_item: ScrapeItem, post: Post, *, is_single_post: bool = False) -> None:
+ async def _handle_post(self, scrape_item: ScrapeItem, post: Post, *, is_single_post: bool = False) -> None:
post_id = str(post.id)
title = self.create_separate_post_title(post.title, post_id, post.date_gmt.date())
if is_single_post:
@@ -149,10 +144,10 @@ class WordPressBaseCrawler(Crawler, is_abc=True):
scrape_item.uploaded_at = to_timestamp(post.date_gmt)
if post.thumbnail:
await self.direct_file(scrape_item, self.parse_url(post.thumbnail))
- return await self.handle_post_content(scrape_item, post)
+ return await self._post_content(scrape_item, post)
@final
- async def handle_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:
+ async def _handle_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:
if self.is_attachment(link):
return await self.direct_file(scrape_item, link)
if self.PRIMARY_URL.host == link.host:
@@ -161,10 +156,10 @@ class WordPressBaseCrawler(Crawler, is_abc=True):
self.handle_external_links(new_scrape_item)
scrape_item.add_children()
- async def handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:
+ async def _post_content(self, scrape_item: ScrapeItem, post: Post) -> None:
for link in self.iter_parse_url(_iter_links(post.content, use_regex=self.WP_USE_REGEX)):
if link:
- await self.handle_link(scrape_item, link)
+ await self._handle_link(scrape_item, link)
@classmethod
def parse_url(
@@ -216,7 +211,7 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):
collections = model.validate_json(await self.request_text(api_url))
if not collections:
raise ScrapeError(404)
- await self.__handle_collection(scrape_item, collections[0], date_range)
+ await self.__collection(scrape_item, collections[0], date_range)
@error_handling_wrapper
async def post(self, scrape_item: ScrapeItem) -> None:
@@ -224,7 +219,7 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):
posts = PostSequence.validate_json(await self.request_text(api_url))
if not posts:
raise ScrapeError(404)
- return await self.handle_post(scrape_item, posts[0], is_single_post=True)
+ return await self._handle_post(scrape_item, posts[0], is_single_post=True)
@error_handling_wrapper
async def all_posts(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:
@@ -232,10 +227,10 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):
if date_range:
api_url = api_url.update_query(date_range.as_query())
scrape_item.setup_as_profile(self.create_title("Posts"))
- async for post in self.post_pager(api_url):
- await self.filter_post(scrape_item, post)
+ async for post in self._post_pager(api_url):
+ await self._filter_post(scrape_item, post)
- async def __handle_collection(
+ async def __collection(
self, scrape_item: ScrapeItem, collection: Category | Tag, date_range: QueryDatetimeRange | None = None
) -> None:
title = self.create_title(f"{collection.description or collection.slug} [{collection._type}]")
@@ -244,10 +239,10 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):
api_url = self.WP_API_POSTS_URL.with_query({collection._type: collection.id})
if date_range:
api_url = api_url.update_query(date_range.as_query())
- async for post in self.post_pager(api_url):
- await self.filter_post(scrape_item, post)
+ async for post in self._post_pager(api_url):
+ await self._filter_post(scrape_item, post)
- async def post_pager(self, url: AbsoluteHttpURL, init_page: int | None = None) -> AsyncIterable[Post]:
+ async def _post_pager(self, url: AbsoluteHttpURL, init_page: int | None = None) -> AsyncIterable[Post]:
for page in itertools.count(init_page or 1):
n_post = 0
api_url = url.update_query(per_page=_POST_PER_REQUEST, page=page)
@@ -279,15 +274,15 @@ class WordPressHTMLCrawler(WordPressBaseCrawler, is_generic=True):
) -> None:
title = self.create_title(f"{collection.description or collection.slug} [{collection._type}]")
scrape_item.setup_as_profile(title)
- await self.post_url_pager(scrape_item, date_range)
+ await self._post_pager(scrape_item, date_range)
@error_handling_wrapper
async def post(self, scrape_item: ScrapeItem) -> None:
soup = await self.__make_request(scrape_item.url)
- post = self.parse_post(scrape_item, soup)
- return await self.handle_post(scrape_item, post, is_single_post=True)
+ post = self._parse_post(scrape_item, soup)
+ return await self._handle_post(scrape_item, post, is_single_post=True)
- def parse_post(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> Post:
+ def _parse_post(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> Post:
title = open_graph.get_title(soup) or css.select_text(soup, Selector.POST_TITLE)
date = open_graph.get("published_time", soup) or _match_date_from_path(scrape_item.url.parts[1:4])
data = {
@@ -303,9 +298,9 @@ class WordPressHTMLCrawler(WordPressBaseCrawler, is_generic=True):
@error_handling_wrapper
async def all_posts(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:
scrape_item.setup_as_profile(self.create_title("Posts"))
- await self.post_url_pager(scrape_item, date_range)
+ await self._post_pager(scrape_item, date_range)
- async def post_url_pager(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:
+ async def _post_pager(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:
async for soup in self.web_pager(scrape_item.url):
for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.POST_LINK_FROM_PAGE.element):
if (
diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.py
index 42155fcd7..cdc628a39 100644
--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py
+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py
@@ -97,9 +97,9 @@ class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):
self.update_cookies(cookies)
credentials = {"login": username, "password": password, "_xfRedirect": str(self.PRIMARY_URL)}
- await self.xf_try_login(login_url, credentials, retries=5)
+ await self._xf_try_login(login_url, credentials, retries=5)
- async def xf_try_login(
+ async def _xf_try_login(
self,
login_url: AbsoluteHttpURL,
credentials: dict[str, str],