Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: adding bluesky support (#2119)
Barbarella666666607d45ede3512
diff --git a/cyberdrop_dl/crawlers/bluesky/__init__.py b/cyberdrop_dl/crawlers/bluesky/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/cyberdrop_dl/crawlers/bluesky/api.py b/cyberdrop_dl/crawlers/bluesky/api.pynew file mode 100644index 000000000..61e225a5a--- /dev/null+++ b/cyberdrop_dl/crawlers/bluesky/api.py@@ -0,0 +1,116 @@+"""https://endpoints.bsky.app+https://github.com/bluesky-social/atproto+"""++from __future__ import annotations++from collections import deque+from typing import TYPE_CHECKING, Any, ClassVar++from cyberdrop_dl import aio+from cyberdrop_dl.crawlers.bluesky.types import FeedFilter, PostView+from cyberdrop_dl.crawlers.crawler import API+from cyberdrop_dl.url_objects import AbsoluteHttpURL++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator, Generator, Iterable+++class BlueSkyAPI(API):+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api.bsky.app/xrpc")+ BLOB_ENDPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://bsky.social/xrpc/com.atproto.sync.getBlob")++ def __post_init__(self) -> None:+ self._did_cache: dict[str, str] = {}+ self._did_locks: aio.WeakAsyncLocks[str] = aio.WeakAsyncLocks()++ @classmethod+ def get_blob(cls, did: str, cid: str) -> AbsoluteHttpURL:+ # did: desentralized ID, cid: content ID (hash)+ return cls.BLOB_ENDPOINT.with_query(did=did, cid=cid)++ async def xrpc(self, path: str, **params: Any) -> dict[str, Any]:+ url = (self.ENTRYPOINT / path).with_query(params)+ return await self.request_json(url)++ async def resolve_handle(self, handle_or_did: str) -> str:+ if handle_or_did.startswith("did:"):+ return handle_or_did+ try:+ return self._did_cache[handle_or_did]+ except LookupError:+ pass++ async with self._did_locks[handle_or_did]:+ try:+ return self._did_cache[handle_or_did]+ except LookupError:+ pass++ resp = await self.xrpc("com.atproto.identity.resolveHandle", handle=handle_or_did)+ did = self._did_cache[handle_or_did] = resp["did"]+ return did++ async def thread(+ self,+ actor: str,+ post_id: str,+ *,+ depth: int = 100, # How many replies+ parent_height: int = 0, # How many parent replies+ ) -> tuple[PostView, Generator[PostView]]:+ did = await self.resolve_handle(actor)+ resp = await self.xrpc(+ "app.bsky.feed.getPostThread",+ uri=f"at://{did}/app.bsky.feed.post/{post_id}",+ depth=depth,+ parentHeight=parent_height,+ )++ thread = resp["thread"]+ og_post = PostView.parse(thread["post"])++ thread_replies: deque[dict[str, Any]] = deque()+ thread_replies.extend(_filter_blocked_replies(thread.get("replies", ())))++ def replies() -> Generator[PostView]:+ while thread_replies:+ reply = thread_replies.popleft()+ yield PostView.parse(reply["post"])+ thread_replies.extend(_filter_blocked_replies(reply.get("replies", ())))++ return og_post, replies()++ async def author_feed(+ self,+ actor: str,+ feed_filter: FeedFilter = "posts_with_media",+ ) -> AsyncGenerator[PostView]:+ async for page in self.pager("app.bsky.feed.getAuthorFeed", actor=actor, filter=feed_filter):+ for post in page:+ yield PostView.parse(post["post"])++ async def pager(+ self,+ path: str,+ key: str = "feed",+ **params: Any,+ ) -> AsyncGenerator[list[dict[str, Any]]]:+ if "actor" in params:+ params["actor"] = await self.resolve_handle(params["actor"])++ params.setdefault("limit", 100)++ while True:+ resp = await self.xrpc(path, **params)+ yield resp[key]+ params["cursor"] = cursor = resp.get("cursor")+ if not cursor:+ return+++def _filter_blocked_replies(replies: Iterable[dict[str, Any]]) -> Generator[dict[str, Any]]:+ for reply in replies:+ if reply.get("not_found") or reply.get("blocked") or reply.get("$type") != "app.bsky.feed.defs#threadViewPost":+ continue+ yield replydiff --git a/cyberdrop_dl/crawlers/bluesky/crawler.py b/cyberdrop_dl/crawlers/bluesky/crawler.pynew file mode 100644index 000000000..cb461c074--- /dev/null+++ b/cyberdrop_dl/crawlers/bluesky/crawler.py@@ -0,0 +1,125 @@+from __future__ import annotations++from typing import TYPE_CHECKING, ClassVar, Literal, cast++from cyberdrop_dl.crawlers.bluesky.api import BlueSkyAPI+from cyberdrop_dl.crawlers.bluesky.types import Media+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths+from cyberdrop_dl.mediaprops import Resolution+from cyberdrop_dl.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils.errors import error_handling_wrapper++if TYPE_CHECKING:+ from collections.abc import Generator++ from cyberdrop_dl.crawlers.bluesky.types import Blob, FeedFilter, Image, MediaAsset, PostView, Video+ from cyberdrop_dl.url_objects import ScrapeItem+++class BlueskyCrawler(Crawler):+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "bsky.app", "bsky.social", "main.bsky.dev"+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Post": "/profile/<handle>/post/<post_id>",+ "Profile": "/profile/<handle>",+ }+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://bsky.app")+ DOMAIN: ClassVar[str] = "bluesky"+ FOLDER_DOMAIN: ClassVar[str] = "BlueSky"+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} - {id}"++ def __post_init__(self) -> None:+ self.api: BlueSkyAPI = BlueSkyAPI.from_crawler(self)++ @property+ def separate_posts(self) -> bool:+ return True++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["profile", user, "post", post_id, *_]:+ await self.post(scrape_item, user, post_id)+ case ["profile", user]:+ await self.user(scrape_item, user, "posts_and_author_threads")+ case _:+ raise ValueError++ @error_handling_wrapper+ async def post(self, scrape_item: ScrapeItem, actor: str, post_id: str) -> None:+ post, replies = await self.api.thread(actor, post_id)+ self._post(scrape_item, post)+ for reply in replies:+ new_item = scrape_item.create_child(self.PRIMARY_URL / reply.web_path)+ self._post(new_item, reply)+ scrape_item.add_children()++ @error_handling_wrapper+ async def user(self, scrape_item: ScrapeItem, actor: str, feed_filter: FeedFilter) -> None:+ scrape_item.setup_as_profile("")+ async for post in self.api.author_feed(actor, feed_filter):+ new_item = scrape_item.create_child(self.PRIMARY_URL / post.web_path)+ self._post(new_item, post)+ scrape_item.add_children()++ @error_handling_wrapper+ def _post(self, scrape_item: ScrapeItem, post: PostView) -> None:+ scrape_item.setup_as_post(self.create_title(f"@{post.author.handle}"))+ scrape_item.uploaded_at = date = self.parse_iso_date(post.record["createdAt"])+ scrape_item.append_folders(self.create_separate_post_title(None, post.id, date))+ self.create_eager_task(self.write_metadata(scrape_item, f"post {post.id}", post))++ for media in _extract_media(post):+ self.create_eager_task(self._media(scrape_item, media, post.author.did))+ scrape_item.add_children()++ async def _media(self, scrape_item: ScrapeItem, media: Media, did: str) -> None:+ src = self.api.get_blob(did, media.cid)+ with self.catch_errors(src):+ name, ext = self.get_filename_and_ext(media.name, mime_type=media.mime)+ await self.handle_file(+ src,+ scrape_item,+ name,+ ext,+ custom_filename=name,+ metadata=media,+ )+++def _extract_media(post: PostView) -> Generator[Media]:+ embed = post.record.get("embed")+ if not embed:+ return++ embed["type"] = embed["$type"]+ embed = cast("MediaAsset", embed) # pyright: ignore[reportInvalidCast]++ match embed["type"]:+ case "app.bsky.embed.images":+ for img in embed["images"]:+ yield parse_asset(img, "image")+ case "app.bsky.embed.gallery":+ for item in embed["items"]:+ yield parse_asset(item, "image" if "image" in item else "video")+ case "app.bsky.embed.image":+ yield parse_asset(embed, "image")+ case "app.bsky.embed.video":+ yield parse_asset(embed, "video")+ case "app.bsky.embed.record" | "app.bsky.embed.recordWithMedia":+ # TODO: handle this+ pass+ case _:+ raise ValueError(embed)+++def parse_asset(asset: Image | Video, key: Literal["image", "video"]) -> Media:+ # TODO: handle video captions+ res = Resolution(**ratio) if (ratio := asset.get("aspectRatio")) else None+ blob: Blob = asset[key] # pyright: ignore[reportGeneralTypeIssues]+ cid = blob["ref"]["$link"]+ return Media(+ type=key,+ cid=cid,+ name=asset.get("alt") or cid,+ mime=blob["mimeType"],+ resolution=res,+ )diff --git a/cyberdrop_dl/crawlers/bluesky/types.py b/cyberdrop_dl/crawlers/bluesky/types.pynew file mode 100644index 000000000..a570d7157--- /dev/null+++ b/cyberdrop_dl/crawlers/bluesky/types.py@@ -0,0 +1,119 @@+"https://github.com/bluesky-social/atproto/blob/727701085829daf8e0440253346b5e70fc3810ad/lexicons/app/bsky/feed/defs.json"++from __future__ import annotations++import dataclasses+from typing import TYPE_CHECKING, Any, Literal, NotRequired, Self, TypedDict++from cyberdrop_dl.models import type_adapter++if TYPE_CHECKING:+ from cyberdrop_dl.mediaprops import Resolution++type FeedFilter = Literal[+ "posts_with_replies",+ "posts_no_replies",+ "posts_with_media",+ "posts_and_author_threads",+ "posts_with_video",+]++[email protected](frozen=True, slots=True)+class ProfileViewBasic:+ did: str+ handle: str+ display_name: str | None = None+ created_at: str | None = None++[email protected](frozen=True, slots=True)+class PostView:+ uri: str+ cid: str+ author: ProfileViewBasic+ record: Record # This is typed as unknown on their spec++ @classmethod+ def parse(cls, data: dict[str, Any]) -> Self:+ return type_adapter(cls).validate_python(data)++ @property+ def id(self) -> str:+ return self.uri.rpartition("/")[-1]++ @property+ def web_path(self) -> str:+ return f"profile/{self.author.handle}/post/{self.id}"+++Href = TypedDict("Href", {"$link": str})+Asset = TypedDict("Asset", {"$type": str})+++class Blob(Asset):+ ref: Href+ mimeType: str # noqa: N815+ size: int+++class AspectRatio(TypedDict):+ width: int+ height: int+++class Image(Asset):+ type: Literal["app.bsky.embed.image"]+ image: Blob+ alt: str+ aspectRatio: AspectRatio | None # noqa: N815+++class Images(Asset):+ type: Literal["app.bsky.embed.images"]+ images: list[Image]+++class Gallery(Asset):+ type: Literal["app.bsky.embed.gallery"]+ items: list[Image]+++class Captions(TypedDict):+ lang: str+ file: Blob+++class Video(Asset):+ type: Literal["app.bsky.embed.video"]+ video: Blob+ alt: str | None+ aspectRatio: AspectRatio | None # noqa: N815+ captions: Captions | None+++class External(Asset):+ type: Literal["app.bsky.embed.external"]+ uri: str+ title: str+ description: str+ thumb: Blob+ created_at: str | None+ updated_at: str | None+++type MediaAsset = Video | Image | External | Images | Gallery+++class Record(TypedDict):+ createdAt: str+ embed: NotRequired[dict[str, Any]]++[email protected](frozen=True, slots=True)+class Media:+ type: Literal["video", "image"]+ cid: str+ name: str+ mime: str+ resolution: Resolution | None = Nonediff --git a/tests/crawlers/test_cases/bluesky.py b/tests/crawlers/test_cases/bluesky.pynew file mode 100644index 000000000..646506834--- /dev/null+++ b/tests/crawlers/test_cases/bluesky.py@@ -0,0 +1,42 @@+DOMAIN = "bluesky"+TEST_CASES = [+ {+ "url": "https://bsky.app/profile/tenzilkemesq.bsky.social/post/3mt5h4og6jc2r",+ "results": [+ {+ "url": "https://bsky.social/xrpc/com.atproto.sync.getBlob?did=did:plc:tbmq3lilsp2ukxhrha76cewi&cid=bafkreidlpgvgk5p5x5hrxwzot2lzy2gwfg2ggbujiuim5gg7adtxwiyeb4",+ "filename": "Footage from Forza Horizon 6 of me breaking a hard 5000 xp bonus board..mp4",+ "debrid_url": None,+ "original_filename": "Footage from Forza Horizon 6 of me breaking a hard 5000 xp bonus board..mp4",+ "referer": "https://bsky.app/profile/tenzilkemesq.bsky.social/post/3mt5h4og6jc2r",+ "album_id": None,+ "uploaded_at": 1786823521,+ "download_folder": "re:@tenzilkemesq.bsky.social (BlueSky)/2026-08-15 - 3mt5h4og6jc2r",+ "thumbnail": None,+ }+ ],+ "count": 1,+ },+ {+ "url": "https://bsky.app/profile/comicsguy67.bsky.social/post/3mupyd6tlyk2x",+ "results": [+ {+ "url": "https://bsky.social/xrpc/com.atproto.sync.getBlob?did=did:plc:gtcw4ogmo7pegvqlcvuay4wv&cid=bafkreib27qhwltdaykcengqzs6k4i2w33u2uuh7ibadf3c5g4eda4owmou",+ "filename": "bafkreib27qhwltdaykcengqzs6k4i2w33u2uuh7ibadf3c5g4eda4owmou.jpg",+ "debrid_url": None,+ "original_filename": "bafkreib27qhwltdaykcengqzs6k4i2w33u2uuh7ibadf3c5g4eda4owmou.jpg",+ "referer": "https://bsky.app/profile/comicsguy67.bsky.social/post/3mupyd6tlyk2x",+ "album_id": None,+ "uploaded_at": 1788559980,+ "download_folder": "re:@comicsguy67.bsky.social (BlueSky)/2026-09-04 - 3mupyd6tlyk2x",+ "thumbnail": None,+ }+ ],+ "count": 1,+ },+ {+ "url": "https://bsky.app/profile/momo951.bsky.social",+ "results": [],+ "count": range(2, 100),+ },+]