Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: add missing @error_handling_wrapper (#1921)
NTFSvolumed8ba654b78ea
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6b2e63e7a..700b2fb5d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -36,6 +36,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Do not crash on HTTP errors from individual photos (Flickr)+- Do not crash on HTTP errors from individual files (Google Drive)+### Security## [9.12.0] - 2026-05-27diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex 627ad7e72..42a5a566b 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -184,27 +184,18 @@ class MessageBoardCrawler(Crawler, is_abc=True):@abstractmethodasync def thread(self, scrape_item: ScrapeItem, /, thread: ThreadProtocol) -> None: ...- async def forum(self, scrape_item: ScrapeItem) -> None:- # Subclasses can define custom logic for this method.- # They would need to also override `fetch` since the default fetch does not take this method into account- raise NotImplementedError-async def resolve_confirmation_link(self, url: AbsoluteHttpURL, /) -> AbsoluteHttpURL | None:# Not every forum has confirmation link so overriding this method is optional# Implementation of this method MUST return `None` instead of raising an errorraise NotImplementedErrorasync def __async_post_init__(self) -> None:- await self.login()-- @final- async def login(self) -> None:if self.login_required is None:returnif not self._logged_in:login_url = self.PRIMARY_URL / "login"- await self._login(login_url)+ await self.login(login_url)@final@property@@ -228,22 +219,22 @@ class MessageBoardCrawler(Crawler, is_abc=True):if self.is_attachment(scrape_item.url):return await self.handle_internal_link(scrape_item)if is_confirmation_link(scrape_item.url):- return await self.follow_confirmation_link(scrape_item)+ return await self._follow_confirmation_link(scrape_item)- await self.fetch_thread(scrape_item)+ await self._fetch_thread(scrape_item)- async def fetch_thread(self, scrape_item: ScrapeItem) -> None:+ async def _fetch_thread(self, scrape_item: ScrapeItem) -> None:thread_part_index = len(self.PRIMARY_URL.parts)# https://github.com/Cyberdrop-DL/cyberdrop-dl/issues/1165#issuecomment-3086739753if self.PRIMARY_URL.parts[-1] == "":thread_part_index -= 1match scrape_item.url.parts[thread_part_index:]:case [thread_part, thread_name_and_id, *_] if thread_part in self.THREAD_PART_NAMES:- self.check_thread_recursion(scrape_item)+ self._check_thread_recursion(scrape_item)thread = self.parse_thread(scrape_item.url, thread_name_and_id)return await self.thread(scrape_item, thread)case ["goto" | "posts", _, *_]:- self.check_thread_recursion(scrape_item)+ self._check_thread_recursion(scrape_item)return await self.follow_redirect(scrape_item)case _:raise ValueError@@ -259,7 +250,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):return by_parts or by_host@final- async def follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:+ async def _follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:url = await self.resolve_confirmation_link(scrape_item.url)if url: # If there was an error, this will be Nonescrape_item.url = url@@ -267,7 +258,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):return self.handle_external_links(scrape_item)@final- def check_thread_recursion(self, scrape_item: ScrapeItem) -> None:+ def _check_thread_recursion(self, scrape_item: ScrapeItem) -> None:if self.stop_thread_recursion(scrape_item):parents = f"{len(scrape_item.parent_threads)} parent thread(s)"msg = (@@ -278,10 +269,10 @@ class MessageBoardCrawler(Crawler, is_abc=True):)raise MaxChildrenError(msg)- self.limit_nexted_thread_folders(scrape_item)+ self._limit_nexted_thread_folders(scrape_item)@final- def limit_nexted_thread_folders(self, scrape_item: ScrapeItem) -> None:+ def _limit_nexted_thread_folders(self, scrape_item: ScrapeItem) -> None:if self.max_thread_folder_depth is None:returnn_parents = len(scrape_item.parent_threads)@@ -325,7 +316,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):await self.handle_file(link, new_scrape_item, filename, ext)@final- async def write_last_forum_post(self, thread_url: AbsoluteHttpURL, last_post_url: AbsoluteHttpURL | None) -> None:+ async def _write_last_forum_post(self, thread_url: AbsoluteHttpURL, last_post_url: AbsoluteHttpURL | None) -> None:if not last_post_url or last_post_url == thread_url:returnself.manager.logs.write_last_post_log(last_post_url)@@ -334,7 +325,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):# TODO: Define an unified workflow for crawlers to perform and check login@final@error_handling_wrapper- async def _login(self, login_url: AbsoluteHttpURL) -> None:+ async def login(self, login_url: AbsoluteHttpURL) -> None:session_cookie = self.get_cookie_value(self.LOGIN_USER_COOKIE_NAME)msg = f"No cookies found for {self.FOLDER_DOMAIN}"if not session_cookie and self.login_required:@@ -422,9 +413,9 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):raise ScrapeError("User Error", msg)self.scraped_threads.add(thread.url)- await self.process_thread(scrape_item, thread)+ await self._thread(scrape_item, thread)- async def process_thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None:+ async def _thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None:title: str = ""last_post_url = thread.urlasync for soup in self.thread_pager(scrape_item):@@ -436,13 +427,13 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):raisescrape_item.add_to_parent_title(title)- continue_scraping, last_post_url = self.process_thread_page(scrape_item, thread, soup)+ continue_scraping, last_post_url = self._thread_page(scrape_item, thread, soup)if not continue_scraping:break- await self.write_last_forum_post(thread.url, last_post_url)+ await self._write_last_forum_post(thread.url, last_post_url)- def process_thread_page(+ def _thread_page(self, scrape_item: ScrapeItem, thread: ThreadProtocol, soup: BeautifulSoup) -> tuple[bool, AbsoluteHttpURL]:continue_scraping = False@@ -576,6 +567,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):new_link = self.parse_url(link_str)return await self.get_absolute_link(new_link)+ @error_handling_wrapperasync def handle_internal_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:link = link or scrape_item.urlslug = link.name or link.parent.namediff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex 13f63c253..3a140038c 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -127,7 +127,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):title = self.create_title(title)scrape_item.setup_as_album(title)- await self.iter_videos(scrape_item)+ await self._iter_videos(scrape_item)@error_handling_wrapperasync def profile(self, scrape_item: ScrapeItem) -> None:@@ -139,13 +139,13 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):scrape_item.setup_as_profile(title)if soup.select_one(Selector.PUBLIC_VIDEOS):- await self.iter_videos(scrape_item, "public_videos")+ await self._iter_videos(scrape_item, "public_videos")if soup.select_one(Selector.FAVOURITE_VIDEOS):- await self.iter_videos(scrape_item, "favourite_videos")+ await self._iter_videos(scrape_item, "favourite_videos")if soup.select_one(Selector.PRIVATE_VIDEOS):- await self.iter_videos(scrape_item, "private_videos")+ await self._iter_videos(scrape_item, "private_videos")- async def iter_videos(self, scrape_item: ScrapeItem, video_category: str = "") -> None:+ async def _iter_videos(self, scrape_item: ScrapeItem, video_category: str = "") -> None:url = scrape_item.url / video_category if video_category else scrape_item.urlasync for soup in self.web_pager(url):for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS):diff --git a/cyberdrop_dl/crawlers/cloudflare_stream.py b/cyberdrop_dl/crawlers/cloudflare_stream.pyindex 038e78974..4077a8c5d 100644--- a/cyberdrop_dl/crawlers/cloudflare_stream.py+++ b/cyberdrop_dl/crawlers/cloudflare_stream.py@@ -43,13 +43,12 @@ class CloudflareStreamCrawler(Crawler):async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case ["embed", _] if video_id := scrape_item.url.query.get("video"):- return await self.video(scrape_item, video_id)+ passcase [video_id, *_]:- return await self.video(scrape_item, video_id)+ passcase _:raise ValueError- async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:# https://developers.cloudflare.com/stream/viewing-videos/securing-your-stream/try:jwt = JSONWebToken.decode(video_id)@@ -63,10 +62,10 @@ class CloudflareStreamCrawler(Crawler):return Nonescrape_item.url = _DEFAULT_VIDEO_CDN / video_id- return await self._video(scrape_item, video_id, token)+ return await self.video(scrape_item, video_id, token)@error_handling_wrapper- async def _video(self, scrape_item: ScrapeItem, video_id: str, token: str | None) -> None:+ async def video(self, scrape_item: ScrapeItem, video_id: str, token: str | None) -> None:if await self.check_complete_from_referer(scrape_item):returndiff --git a/cyberdrop_dl/crawlers/discourse/__init__.py b/cyberdrop_dl/crawlers/discourse/__init__.pyindex ef84a1b38..132c82d36 100644--- a/cyberdrop_dl/crawlers/discourse/__init__.py+++ b/cyberdrop_dl/crawlers/discourse/__init__.py@@ -49,6 +49,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):def is_attachment(cls, link: AbsoluteHttpURL | str) -> bool:return "uploads" in AbsoluteHttpURL(link).parts+ @error_handling_wrapperasync def handle_internal_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:link = link or scrape_item.urlif len(link.parts) < 5 and not link.suffix:@@ -92,6 +93,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):def parse_thread(cls, url: AbsoluteHttpURL, thread_name_and_id: str) -> None: # pyright: ignore[reportIncompatibleMethodOverride]raise NotImplementedError+ @error_handling_wrapperasync def thread(self, scrape_item: ScrapeItem, /, thread: Topic) -> None: # pyright: ignore[reportIncompatibleMethodOverride]await self.topic(scrape_item, thread)@@ -111,7 +113,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):if last_post_id:topic_url = self.PRIMARY_URL / topic.path.removeprefix("/")post_url = topic_url / str(last_post_id)- await self.write_last_forum_post(topic_url, post_url)+ await self._write_last_forum_post(topic_url, post_url)async def iter_posts(self, topic: Topic) -> AsyncIterable[AvailablePost]:for offset in itertools.count(topic.init_post_number - 1, _MAX_POSTS_PER_REQUEST):@@ -124,6 +126,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):if topic.init_post_number != 1 and self.scrape_single_forum_post:return+ @error_handling_wrapperasync def post(self, scrape_item: ScrapeItem, /, post: AvailablePost) -> None: # pyright: ignore[reportIncompatibleMethodOverride]title = self.create_separate_post_title(post.title, str(post.id), post.created_at)scrape_item.setup_as_post(title)diff --git a/cyberdrop_dl/crawlers/flickr.py b/cyberdrop_dl/crawlers/flickr.pyindex a96a36443..18ee94ec4 100644--- a/cyberdrop_dl/crawlers/flickr.py+++ b/cyberdrop_dl/crawlers/flickr.py@@ -58,6 +58,7 @@ class FlickrCrawler(Crawler):self.create_task(self._photo(new_scrape_item, photo))scrape_item.add_children()+ @error_handling_wrapperasync def photo(self, scrape_item: ScrapeItem, photo_id: str) -> None:if await self.check_complete_from_referer(scrape_item):returndiff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.pyindex 9f44747b2..c28b14bdf 100644--- a/cyberdrop_dl/crawlers/google_drive.py+++ b/cyberdrop_dl/crawlers/google_drive.py@@ -130,6 +130,7 @@ class GoogleDriveCrawler(Crawler):if index % 200 == 0:await asyncio.sleep(0)+ @error_handling_wrapperasync def file(self, scrape_item: ScrapeItem, file_id: str = "", doc: str | None = None) -> None:version = int(file_id[0])if version not in _KNOWN_FILE_ID_VERSIONS:diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex fdc5fbbb3..667bce2df 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -64,7 +64,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):def password(self) -> str | None:return self.manager.config.auth.meganz.password or None- async def __async_post_init__(self) -> None:+ def __post_init__(self) -> None:self._decryption_keys: dict[AbsoluteHttpURL, tuple[Crypto, int]] = {}api = MegaAPI(self.manager.http_client._session)api.user_agent = CDL_USER_AGENT@@ -72,7 +72,9 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):speed_limiter = self.downloader.client.speed_limiterself.downloader = MegaDownloader(self.manager, self.DOMAIN) # pyright: ignore[reportIncompatibleVariableOverride]self.downloader.client.speed_limiter = speed_limiter- await self.login()++ async def __async_post_init__(self) -> None:+ await self._login()async def fetch(self, scrape_item: ScrapeItem) -> None:if not self._logged_in:@@ -163,7 +165,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):media_item.extra_info.setdefault(self.DOMAIN, {})["key"] = self._decryption_keys.pop(media_item.url)await super().handle_media_item(media_item, m3u8)- async def login(self) -> None:+ async def _login(self) -> None:# This takes a really long time (dozens of seconds)# TODO: Add a way to cache this login# TODO: Show some logging message / UI about logindiff --git a/cyberdrop_dl/crawlers/nsfw_xxx.py b/cyberdrop_dl/crawlers/nsfw_xxx.pyindex 29e96bebe..5b1fe63ac 100644--- a/cyberdrop_dl/crawlers/nsfw_xxx.py+++ b/cyberdrop_dl/crawlers/nsfw_xxx.py@@ -48,23 +48,26 @@ class NsfwXXXCrawler(Crawler):case _:raise ValueError+ @error_handling_wrapperasync def subreddit(self, scrape_item: ScrapeItem, subreddit: str) -> None:api_url = self.PRIMARY_URL / "api/v1/source/r" / subredditawait self._collection(scrape_item, api_url)+ @error_handling_wrapperasync def category(self, scrape_item: ScrapeItem, name: str) -> None:api_url = self.PRIMARY_URL / "api/v1/category" / nameawait self._collection(scrape_item, api_url)+ @error_handling_wrapperasync def search(self, scrape_item: ScrapeItem, query: str) -> None:api_url = (self.PRIMARY_URL / "api/v1/search").with_query(q=query)await self._collection(scrape_item, api_url, query)+ @error_handling_wrapperasync def user(self, scrape_item: ScrapeItem, username: str) -> None:api_url = self.PRIMARY_URL / "api/v1/user" / usernameawait self._collection(scrape_item, api_url, f"@{username}")- @error_handling_wrapperasync def _collection(self, scrape_item: ScrapeItem, api_url: AbsoluteHttpURL, name: str | None = None) -> None:title: str = ""type_ = api_url.parts[3]diff --git a/cyberdrop_dl/crawlers/pcloud.py b/cyberdrop_dl/crawlers/pcloud.pyindex bfe83c78e..c07a87d42 100644--- a/cyberdrop_dl/crawlers/pcloud.py+++ b/cyberdrop_dl/crawlers/pcloud.py@@ -88,7 +88,7 @@ class PCloudCrawler(Crawler):self._iter_nodes(scrape_item, node.contents)return None- return await self.file(scrape_item, cast("File", node))+ return await self._file(scrape_item, cast("File", node))def _iter_nodes(self, scrape_item: ScrapeItem, nodes: Sequence[Node], *parents: str) -> None:folders: list[Node] = []@@ -109,7 +109,7 @@ class PCloudCrawler(Crawler):for folder in folders:self._iter_nodes(scrape_item, folder.contents, *parents, folder.name)- async def file(self, scrape_item: ScrapeItem, file: File) -> None:+ async def _file(self, scrape_item: ScrapeItem, file: File) -> None:# https://docs.pcloud.com/methods/public_links/getpublinkdownload.htmllink = await self._request_download_url(scrape_item, file)@@ -120,7 +120,7 @@ class PCloudCrawler(Crawler):db_url = (scrape_item.url.origin() / "file" / file._id).with_query(code=scrape_item.url.query["code"])await self.handle_file(db_url, scrape_item, file.name, ext, debrid_link=link, custom_filename=filename)- _file_task = auto_task_id(error_handling_wrapper(file))+ _file_task = auto_task_id(error_handling_wrapper(_file))async def _request_download_url(self, scrape_item: ScrapeItem, file: File) -> AbsoluteHttpURL:path = "getmediatranscodepublink" if "video" in file.contenttype else "getpublinkdownload"diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.pyindex 83d09948b..ef228b567 100644--- a/cyberdrop_dl/crawlers/pkmncards.py+++ b/cyberdrop_dl/crawlers/pkmncards.py@@ -127,7 +127,7 @@ class PkmncardsCrawler(Crawler):download_url = self.parse_url(link_str)simple_card = create_simple_card(title, download_url)new_scrape_item = scrape_item.create_child(card_page_url)- self.create_task(self.handle_simple_card(new_scrape_item, simple_card))+ self.create_task(self._simple_card(new_scrape_item, simple_card))scrape_item.add_children()@error_handling_wrapper@@ -155,7 +155,7 @@ class PkmncardsCrawler(Crawler):custom_filename = self.create_custom_filename(card.full_name, ext)await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)- async def handle_simple_card(self, scrape_item: ScrapeItem, simple_card: SimpleCard) -> None:+ async def _simple_card(self, scrape_item: ScrapeItem, simple_card: SimpleCard) -> None:@error_handling_wrapperasync def get_card_set(self, scrape_item: ScrapeItem) -> CardSet:soup = await self.request_soup(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/pornpics.py b/cyberdrop_dl/crawlers/pornpics.pyindex 5c5758f41..54cc15f3f 100644--- a/cyberdrop_dl/crawlers/pornpics.py+++ b/cyberdrop_dl/crawlers/pornpics.py@@ -88,6 +88,7 @@ class PornPicsCrawler(Crawler):filename, ext = self.get_filename_and_ext(new_scrape_item.url.name)await self.handle_file(new_scrape_item.url, new_scrape_item, filename, ext)+ @error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:scrape_item.album_id = scrape_item.url.parts[-2]await self.direct_file(scrape_item)diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.pyindex 689f1c8e7..b3828599a 100644--- a/cyberdrop_dl/crawlers/porntrex.py+++ b/cyberdrop_dl/crawlers/porntrex.py@@ -126,11 +126,11 @@ class PorntrexCrawler(Crawler):for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.VIDEOS_OR_ALBUMS):self.create_task(self.run(new_scrape_item))- await self.proccess_additional_pages(scrape_item, last_page)+ await self._ajax_pagination(scrape_item, last_page)if "models" in scrape_item.url.parts:# Additional album pages- await self.proccess_additional_pages(scrape_item, last_page, block_id="list_albums_common_albums_list")+ await self._ajax_pagination(scrape_item, last_page, block_id="list_albums_common_albums_list")elif "members" in scrape_item.url.parts:albums_url = scrape_item.url / "albums"@@ -139,7 +139,7 @@ class PorntrexCrawler(Crawler):for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ALBUMS, new_title_part="albums"):self.create_task(self.run(new_scrape_item))- async def proccess_additional_pages(self, scrape_item: ScrapeItem, last_page: int, **kwargs: str) -> None: # noqa: C901+ async def _ajax_pagination(self, scrape_item: ScrapeItem, last_page: int, **kwargs: str) -> None: # noqa: C901if last_page == 1:returnblock_id: str = "list_videos_common_videos_list_norm"diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.pyindex e3ca5e32f..dc1eaf79e 100644--- a/cyberdrop_dl/crawlers/sendvid.py+++ b/cyberdrop_dl/crawlers/sendvid.py@@ -27,7 +27,7 @@ class SendVidCrawler(Crawler):FOLDER_DOMAIN: ClassVar[str] = "SendVid"async def fetch(self, scrape_item: ScrapeItem) -> None:- scrape_item.url = self.get_streaming_url(scrape_item.url)+ scrape_item.url = _get_streaming_url(scrape_item.url)await self.video(scrape_item)@error_handling_wrapper@@ -43,11 +43,9 @@ class SendVidCrawler(Crawler):except AssertionError:raise ScrapeError(422, "Couldn't find video source") from Nonelink = self.parse_url(link_str)- await self.handle_direct_link(scrape_item, link, title)+ await self._video(scrape_item, link, title)- async def handle_direct_link(- self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None, title: str = ""- ) -> None:+ async def _video(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None, title: str = "") -> None:link = link or scrape_item.urlcanonical_url = link.with_query(None)@@ -61,14 +59,15 @@ class SendVidCrawler(Crawler):canonical_url, scrape_item, filename, ext, debrid_link=link, custom_filename=custom_filename)- def get_streaming_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:- if is_cdn(url):- video_id = url.name.split(".", 1)[0]- return PRIMARY_URL.with_path(video_id)- if "embed" in url.parts:- return url.with_path(url.path.replace("/embed/", "/"), keep_query=True, keep_fragment=True)- return url+def _get_streaming_url(url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ if is_cdn(url):+ video_id = url.name.split(".", 1)[0]+ return PRIMARY_URL.with_path(video_id)++ if "embed" in url.parts:+ return url.with_path(url.path.replace("/embed/", "/"), keep_query=True, keep_fragment=True)+ return urldef is_cdn(url: AbsoluteHttpURL) -> bool:diff --git a/cyberdrop_dl/crawlers/sex_dot_com.py b/cyberdrop_dl/crawlers/sex_dot_com.pyindex 856f23698..bdbb4db90 100644--- a/cyberdrop_dl/crawlers/sex_dot_com.py+++ b/cyberdrop_dl/crawlers/sex_dot_com.py@@ -55,7 +55,7 @@ class SexDotComCrawler(Crawler):return (await self.request_json(data_url))["media"]@error_handling_wrapper- async def handle_media(self, scrape_item: ScrapeItem, item: dict[str, Any] | None) -> None:+ async def _media(self, scrape_item: ScrapeItem, item: dict[str, Any] | None) -> None:real_item = item or await self.get_media(scrape_item)relative_url = real_item["relativeUrl"]canonical_url = _SHORTS_URL / relative_url@@ -82,10 +82,11 @@ class SexDotComCrawler(Crawler):async def profile(self, scrape_item: ScrapeItem) -> None:async for json_data in self.shorts_profile_paginator(scrape_item):for item in json_data["page"]["items"]:- await self.handle_media(scrape_item, item["media"])+ await self._media(scrape_item, item["media"])+ @error_handling_wrapperasync def post(self, scrape_item: ScrapeItem) -> None:username = scrape_item.url.parts[2]title = self.create_title(username)scrape_item.setup_as_album(title)- await self.handle_media(scrape_item, None)+ await self._media(scrape_item, None)diff --git a/cyberdrop_dl/crawlers/twitter_images.py b/cyberdrop_dl/crawlers/twitter_images.pyindex d629b4518..0f4fd23ea 100644--- a/cyberdrop_dl/crawlers/twitter_images.py+++ b/cyberdrop_dl/crawlers/twitter_images.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem+from cyberdrop_dl.utils import error_handling_wrapperif TYPE_CHECKING:from collections.abc import Generator@@ -28,6 +29,7 @@ class TwimgCrawler(Crawler):return await self.direct_file(scrape_item)await self.photo(scrape_item)+ @error_handling_wrapperasync def photo(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:# https://developer.x.com/en/docs/x-api/v1/data-dictionary/object-model/entities#photo_formatlink = url or scrape_item.urldiff --git a/cyberdrop_dl/crawlers/vbulletin/__init__.py b/cyberdrop_dl/crawlers/vbulletin/__init__.pyindex b27e65d9e..678497c28 100644--- a/cyberdrop_dl/crawlers/vbulletin/__init__.py+++ b/cyberdrop_dl/crawlers/vbulletin/__init__.py@@ -9,6 +9,7 @@ from typing_extensions import overridefrom cyberdrop_dl.crawlers.xenforo.xenforo import XenforoCrawlerfrom cyberdrop_dl.exceptions import MaxChildrenError, ScrapeError+from cyberdrop_dl.utils import error_handling_wrapperif TYPE_CHECKING:import datetime@@ -70,7 +71,7 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801async def __async_post_init__(self) -> None:if not self._logged_in:login_url = self.PRIMARY_URL / "login.php"- await self._login(login_url)+ await self.login(login_url)@overrideasync def check_login_with_request(self, *_: object, **_kwargs: object) -> tuple[str, bool]:@@ -81,25 +82,26 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801# TODO: Handle more URLsif self.login_required and not self._logged_in:return- await self.fetch_thread(scrape_item)+ await self._fetch_thread(scrape_item)- async def process_thread(self, scrape_item: ScrapeItem, thread: Thread) -> None: # pyright: ignore[reportIncompatibleMethodOverride]+ @override+ async def _thread(self, scrape_item: ScrapeItem, thread: Thread) -> None: # pyright: ignore[reportIncompatibleMethodOverride]title: str = ""if thread.post_id and self.scrape_single_forum_post:api_url = self.VBULLETIN_API_ENDPOINT.with_query({self.VBULLETIN_POST_QUERY_PARAM: str(thread.post_id)})else:api_url = self.VBULLETIN_API_ENDPOINT.with_query({self.VBULLETIN_THREAD_QUERY_PARAM: str(thread.id)})- root_xml = await self.get_xml(api_url)+ root_xml = _parse_xml(await self.request_text(api_url))if (thread_element := root_xml.find("thread")) is not None:title = self.create_title(thread_element.attrib["title"], thread_id=thread.id)scrape_item.setup_as_forum(title)else:raise ScrapeError(422)- await self.process_posts(scrape_item, thread, root_xml)+ await self._posts(scrape_item, thread, root_xml)- async def process_posts(self, scrape_item: ScrapeItem, thread: Thread, root_xml: ET.Element[str]) -> None:+ async def _posts(self, scrape_item: ScrapeItem, thread: Thread, root_xml: ET.Element[str]) -> None:if thread.page:posts = itertools.islice(root_xml.iter("post"), (thread.page - 1) * _N_POSTS_PER_PAGE)else:@@ -122,8 +124,10 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801if last_post_id:last_post_url = thread.url.update_query({self.VBULLETIN_POST_QUERY_PARAM: str(last_post_id)})- await self.write_last_forum_post(thread.url, last_post_url)+ await self._write_last_forum_post(thread.url, last_post_url)+ @override+ @error_handling_wrapperasync def post(self, scrape_item: ScrapeItem, post: Post) -> None: # pyright: ignore[reportIncompatibleMethodOverride]title = self.create_separate_post_title(post.title, str(post.id), None)scrape_item.setup_as_post(title)@@ -132,10 +136,11 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801self.handle_external_links(new_scrap_item)scrape_item.add_children()- async def get_xml(self, url: AbsoluteHttpURL) -> ET.Element[str]:- root_xml = ET.XML(await self.request_text(url))- if error := root_xml.find("error"):- details = error.attrib["details"]- error_code = 403 if error.attrib["type"] == "permissions" and "unknown" not in details.casefold() else 422- raise ScrapeError(error_code, message=details)- return root_xml++def _parse_xml(content: str) -> ET.Element[str]:+ root_xml = ET.XML(content)+ if error := root_xml.find("error"):+ details = error.attrib["details"]+ error_code = 403 if error.attrib["type"] == "permissions" and "unknown" not in details.casefold() else 422+ raise ScrapeError(error_code, message=details)+ return root_xmldiff --git a/cyberdrop_dl/crawlers/voe_sx.py b/cyberdrop_dl/crawlers/voe_sx.pyindex 1f65f7473..8219a3fee 100644--- a/cyberdrop_dl/crawlers/voe_sx.py+++ b/cyberdrop_dl/crawlers/voe_sx.py@@ -92,11 +92,7 @@ class VoeSxCrawler(Crawler):async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:- case ["e", video_id]:- return await self.embed(scrape_item, video_id)- case [video_id, "download"]:- return await self.embed(scrape_item, video_id)- case [video_id]:+ case ["e", video_id] | [video_id, "download"] | [video_id]:return await self.embed(scrape_item, video_id)case _:raise ValueError@@ -112,7 +108,7 @@ class VoeSxCrawler(Crawler):text = await resp.text()if match := _find_js_redirect(text):scrape_item.url = self.parse_url(match.group(1), origin)- self.create_task(self.redirect(scrape_item))+ self.create_task(self._redirect(scrape_item))returnsoup = await resp.soup()@@ -153,7 +149,7 @@ class VoeSxCrawler(Crawler):self.handle_subs(scrape_item, custom_filename, video.subtitles)- redirect = auto_task_id(fetch)+ _redirect = auto_task_id(fetch)def extract_voe_video(soup: BeautifulSoup, origin: AbsoluteHttpURL) -> VoeVideo:diff --git a/cyberdrop_dl/crawlers/wordpress/__init__.py b/cyberdrop_dl/crawlers/wordpress/__init__.pyindex 0c42332d4..5167fda99 100644--- a/cyberdrop_dl/crawlers/wordpress/__init__.py+++ b/cyberdrop_dl/crawlers/wordpress/__init__.py@@ -79,7 +79,6 @@ class WordPressBaseCrawler(Crawler, is_abc=True):def __init_subclass__(cls, **kwargs: Any) -> None:super().__init_subclass__(**kwargs)assert cls.fetch is WordPressBaseCrawler.fetch- assert cls.fetch_with_date_range is WordPressBaseCrawler.fetch_with_date_range@finalasync def fetch(self, scrape_item: ScrapeItem) -> None:@@ -87,7 +86,20 @@ class WordPressBaseCrawler(Crawler, is_abc=True):scrape_item.url = scrape_item.url.with_query(None)if date_range:self.log.info(f"Scraping {scrape_item.url} with date range: {date_range.as_query()}")- return await self.fetch_with_date_range(scrape_item, date_range)+ match scrape_item.url.parts[1:]:+ case ["posts"]:+ return await self.all_posts(scrape_item, date_range)+ case [ColletionType.CATEGORY.value, _]:+ return await self.category_or_tag(scrape_item, ColletionType.CATEGORY, date_range)+ case [ColletionType.TAG.value, _]:+ return await self.category_or_tag(scrape_item, ColletionType.TAG, date_range)+ case ["wp-json", *_]:+ raise ValueError+ case _:+ if _match_date_from_path(scrape_item.url.parts[1:]):+ # TODO: Handle this+ raise ValueError+ return await self.post(scrape_item)@property@final@@ -100,23 +112,6 @@ class WordPressBaseCrawler(Crawler, is_abc=True):def is_attachment(url: AbsoluteHttpURL) -> bool:return "wp-content" in url.parts and bool(url.suffix)- @final- async def fetch_with_date_range(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None) -> None:- match scrape_item.url.parts[1:]:- case ["posts"]:- return await self.all_posts(scrape_item, date_range)- case [ColletionType.CATEGORY.value, _]:- return await self.category_or_tag(scrape_item, ColletionType.CATEGORY, date_range)- case [ColletionType.TAG.value, _]:- return await self.category_or_tag(scrape_item, ColletionType.TAG, date_range)- case ["wp-json", *_]:- raise ValueError-- if _ := _match_date_from_path(scrape_item.url.parts[1:]):- # TODO: Handle this- raise ValueError- return await self.post(scrape_item)-@abstractmethodasync def category_or_tag(self, scrape_item: ScrapeItem, /, colletion_type: ColletionType, date_range: QueryDatetimeRange | None = None@@ -129,18 +124,18 @@ class WordPressBaseCrawler(Crawler, is_abc=True):async def all_posts(self, scrape_item: ScrapeItem, /, date_range: QueryDatetimeRange | None = None) -> None: ...@final- async def filter_post(+ async def _filter_post(self, scrape_item: ScrapeItem, post: Post, date_range: QueryDatetimeRange | None = None) -> None:if date_range and not date_range.is_in_range(post.date_gmt):self.log.info(f"Skipping post {post.link} as it is out of date range. Post date: {[post.date_gmt]}")returnnew_scrape_item = scrape_item.create_child(self.parse_url(post.link))- await self.handle_post(new_scrape_item, post)+ await self._handle_post(new_scrape_item, post)scrape_item.add_children()@final- async def handle_post(self, scrape_item: ScrapeItem, post: Post, *, is_single_post: bool = False) -> None:+ async def _handle_post(self, scrape_item: ScrapeItem, post: Post, *, is_single_post: bool = False) -> None:post_id = str(post.id)title = self.create_separate_post_title(post.title, post_id, post.date_gmt.date())if is_single_post:@@ -149,10 +144,10 @@ class WordPressBaseCrawler(Crawler, is_abc=True):scrape_item.uploaded_at = to_timestamp(post.date_gmt)if post.thumbnail:await self.direct_file(scrape_item, self.parse_url(post.thumbnail))- return await self.handle_post_content(scrape_item, post)+ return await self._post_content(scrape_item, post)@final- async def handle_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:+ async def _handle_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:if self.is_attachment(link):return await self.direct_file(scrape_item, link)if self.PRIMARY_URL.host == link.host:@@ -161,10 +156,10 @@ class WordPressBaseCrawler(Crawler, is_abc=True):self.handle_external_links(new_scrape_item)scrape_item.add_children()- async def handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:+ async def _post_content(self, scrape_item: ScrapeItem, post: Post) -> None:for link in self.iter_parse_url(_iter_links(post.content, use_regex=self.WP_USE_REGEX)):if link:- await self.handle_link(scrape_item, link)+ await self._handle_link(scrape_item, link)@classmethoddef parse_url(@@ -216,7 +211,7 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):collections = model.validate_json(await self.request_text(api_url))if not collections:raise ScrapeError(404)- await self.__handle_collection(scrape_item, collections[0], date_range)+ await self.__collection(scrape_item, collections[0], date_range)@error_handling_wrapperasync def post(self, scrape_item: ScrapeItem) -> None:@@ -224,7 +219,7 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):posts = PostSequence.validate_json(await self.request_text(api_url))if not posts:raise ScrapeError(404)- return await self.handle_post(scrape_item, posts[0], is_single_post=True)+ return await self._handle_post(scrape_item, posts[0], is_single_post=True)@error_handling_wrapperasync def all_posts(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:@@ -232,10 +227,10 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):if date_range:api_url = api_url.update_query(date_range.as_query())scrape_item.setup_as_profile(self.create_title("Posts"))- async for post in self.post_pager(api_url):- await self.filter_post(scrape_item, post)+ async for post in self._post_pager(api_url):+ await self._filter_post(scrape_item, post)- async def __handle_collection(+ async def __collection(self, scrape_item: ScrapeItem, collection: Category | Tag, date_range: QueryDatetimeRange | None = None) -> None:title = self.create_title(f"{collection.description or collection.slug} [{collection._type}]")@@ -244,10 +239,10 @@ class WordPressMediaCrawler(WordPressBaseCrawler, is_generic=True):api_url = self.WP_API_POSTS_URL.with_query({collection._type: collection.id})if date_range:api_url = api_url.update_query(date_range.as_query())- async for post in self.post_pager(api_url):- await self.filter_post(scrape_item, post)+ async for post in self._post_pager(api_url):+ await self._filter_post(scrape_item, post)- async def post_pager(self, url: AbsoluteHttpURL, init_page: int | None = None) -> AsyncIterable[Post]:+ async def _post_pager(self, url: AbsoluteHttpURL, init_page: int | None = None) -> AsyncIterable[Post]:for page in itertools.count(init_page or 1):n_post = 0api_url = url.update_query(per_page=_POST_PER_REQUEST, page=page)@@ -279,15 +274,15 @@ class WordPressHTMLCrawler(WordPressBaseCrawler, is_generic=True):) -> None:title = self.create_title(f"{collection.description or collection.slug} [{collection._type}]")scrape_item.setup_as_profile(title)- await self.post_url_pager(scrape_item, date_range)+ await self._post_pager(scrape_item, date_range)@error_handling_wrapperasync def post(self, scrape_item: ScrapeItem) -> None:soup = await self.__make_request(scrape_item.url)- post = self.parse_post(scrape_item, soup)- return await self.handle_post(scrape_item, post, is_single_post=True)+ post = self._parse_post(scrape_item, soup)+ return await self._handle_post(scrape_item, post, is_single_post=True)- def parse_post(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> Post:+ def _parse_post(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> Post:title = open_graph.get_title(soup) or css.select_text(soup, Selector.POST_TITLE)date = open_graph.get("published_time", soup) or _match_date_from_path(scrape_item.url.parts[1:4])data = {@@ -303,9 +298,9 @@ class WordPressHTMLCrawler(WordPressBaseCrawler, is_generic=True):@error_handling_wrapperasync def all_posts(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:scrape_item.setup_as_profile(self.create_title("Posts"))- await self.post_url_pager(scrape_item, date_range)+ await self._post_pager(scrape_item, date_range)- async def post_url_pager(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:+ async def _post_pager(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:async for soup in self.web_pager(scrape_item.url):for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.POST_LINK_FROM_PAGE.element):if (diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.pyindex 42155fcd7..cdc628a39 100644--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py@@ -97,9 +97,9 @@ class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):self.update_cookies(cookies)credentials = {"login": username, "password": password, "_xfRedirect": str(self.PRIMARY_URL)}- await self.xf_try_login(login_url, credentials, retries=5)+ await self._xf_try_login(login_url, credentials, retries=5)- async def xf_try_login(+ async def _xf_try_login(self,login_url: AbsoluteHttpURL,credentials: dict[str, str],