Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: adding bluesky support (#2119)

Barbarella666666607d45ede3512
diff --git a/cyberdrop_dl/crawlers/bluesky/__init__.py b/cyberdrop_dl/crawlers/bluesky/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/cyberdrop_dl/crawlers/bluesky/api.py b/cyberdrop_dl/crawlers/bluesky/api.py
new file mode 100644
index 000000000..61e225a5a
--- /dev/null
+++ b/cyberdrop_dl/crawlers/bluesky/api.py
@@ -0,0 +1,116 @@
+"""https://endpoints.bsky.app
+https://github.com/bluesky-social/atproto
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from cyberdrop_dl import aio
+from cyberdrop_dl.crawlers.bluesky.types import FeedFilter, PostView
+from cyberdrop_dl.crawlers.crawler import API
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator, Generator, Iterable
+
+
+class BlueSkyAPI(API):
+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api.bsky.app/xrpc")
+ BLOB_ENDPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://bsky.social/xrpc/com.atproto.sync.getBlob")
+
+ def __post_init__(self) -> None:
+ self._did_cache: dict[str, str] = {}
+ self._did_locks: aio.WeakAsyncLocks[str] = aio.WeakAsyncLocks()
+
+ @classmethod
+ def get_blob(cls, did: str, cid: str) -> AbsoluteHttpURL:
+ # did: desentralized ID, cid: content ID (hash)
+ return cls.BLOB_ENDPOINT.with_query(did=did, cid=cid)
+
+ async def xrpc(self, path: str, **params: Any) -> dict[str, Any]:
+ url = (self.ENTRYPOINT / path).with_query(params)
+ return await self.request_json(url)
+
+ async def resolve_handle(self, handle_or_did: str) -> str:
+ if handle_or_did.startswith("did:"):
+ return handle_or_did
+ try:
+ return self._did_cache[handle_or_did]
+ except LookupError:
+ pass
+
+ async with self._did_locks[handle_or_did]:
+ try:
+ return self._did_cache[handle_or_did]
+ except LookupError:
+ pass
+
+ resp = await self.xrpc("com.atproto.identity.resolveHandle", handle=handle_or_did)
+ did = self._did_cache[handle_or_did] = resp["did"]
+ return did
+
+ async def thread(
+ self,
+ actor: str,
+ post_id: str,
+ *,
+ depth: int = 100, # How many replies
+ parent_height: int = 0, # How many parent replies
+ ) -> tuple[PostView, Generator[PostView]]:
+ did = await self.resolve_handle(actor)
+ resp = await self.xrpc(
+ "app.bsky.feed.getPostThread",
+ uri=f"at://{did}/app.bsky.feed.post/{post_id}",
+ depth=depth,
+ parentHeight=parent_height,
+ )
+
+ thread = resp["thread"]
+ og_post = PostView.parse(thread["post"])
+
+ thread_replies: deque[dict[str, Any]] = deque()
+ thread_replies.extend(_filter_blocked_replies(thread.get("replies", ())))
+
+ def replies() -> Generator[PostView]:
+ while thread_replies:
+ reply = thread_replies.popleft()
+ yield PostView.parse(reply["post"])
+ thread_replies.extend(_filter_blocked_replies(reply.get("replies", ())))
+
+ return og_post, replies()
+
+ async def author_feed(
+ self,
+ actor: str,
+ feed_filter: FeedFilter = "posts_with_media",
+ ) -> AsyncGenerator[PostView]:
+ async for page in self.pager("app.bsky.feed.getAuthorFeed", actor=actor, filter=feed_filter):
+ for post in page:
+ yield PostView.parse(post["post"])
+
+ async def pager(
+ self,
+ path: str,
+ key: str = "feed",
+ **params: Any,
+ ) -> AsyncGenerator[list[dict[str, Any]]]:
+ if "actor" in params:
+ params["actor"] = await self.resolve_handle(params["actor"])
+
+ params.setdefault("limit", 100)
+
+ while True:
+ resp = await self.xrpc(path, **params)
+ yield resp[key]
+ params["cursor"] = cursor = resp.get("cursor")
+ if not cursor:
+ return
+
+
+def _filter_blocked_replies(replies: Iterable[dict[str, Any]]) -> Generator[dict[str, Any]]:
+ for reply in replies:
+ if reply.get("not_found") or reply.get("blocked") or reply.get("$type") != "app.bsky.feed.defs#threadViewPost":
+ continue
+ yield reply
diff --git a/cyberdrop_dl/crawlers/bluesky/crawler.py b/cyberdrop_dl/crawlers/bluesky/crawler.py
new file mode 100644
index 000000000..cb461c074
--- /dev/null
+++ b/cyberdrop_dl/crawlers/bluesky/crawler.py
@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar, Literal, cast
+
+from cyberdrop_dl.crawlers.bluesky.api import BlueSkyAPI
+from cyberdrop_dl.crawlers.bluesky.types import Media
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
+from cyberdrop_dl.mediaprops import Resolution
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils.errors import error_handling_wrapper
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
+
+ from cyberdrop_dl.crawlers.bluesky.types import Blob, FeedFilter, Image, MediaAsset, PostView, Video
+ from cyberdrop_dl.url_objects import ScrapeItem
+
+
+class BlueskyCrawler(Crawler):
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "bsky.app", "bsky.social", "main.bsky.dev"
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Post": "/profile/<handle>/post/<post_id>",
+ "Profile": "/profile/<handle>",
+ }
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://bsky.app")
+ DOMAIN: ClassVar[str] = "bluesky"
+ FOLDER_DOMAIN: ClassVar[str] = "BlueSky"
+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} - {id}"
+
+ def __post_init__(self) -> None:
+ self.api: BlueSkyAPI = BlueSkyAPI.from_crawler(self)
+
+ @property
+ def separate_posts(self) -> bool:
+ return True
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["profile", user, "post", post_id, *_]:
+ await self.post(scrape_item, user, post_id)
+ case ["profile", user]:
+ await self.user(scrape_item, user, "posts_and_author_threads")
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def post(self, scrape_item: ScrapeItem, actor: str, post_id: str) -> None:
+ post, replies = await self.api.thread(actor, post_id)
+ self._post(scrape_item, post)
+ for reply in replies:
+ new_item = scrape_item.create_child(self.PRIMARY_URL / reply.web_path)
+ self._post(new_item, reply)
+ scrape_item.add_children()
+
+ @error_handling_wrapper
+ async def user(self, scrape_item: ScrapeItem, actor: str, feed_filter: FeedFilter) -> None:
+ scrape_item.setup_as_profile("")
+ async for post in self.api.author_feed(actor, feed_filter):
+ new_item = scrape_item.create_child(self.PRIMARY_URL / post.web_path)
+ self._post(new_item, post)
+ scrape_item.add_children()
+
+ @error_handling_wrapper
+ def _post(self, scrape_item: ScrapeItem, post: PostView) -> None:
+ scrape_item.setup_as_post(self.create_title(f"@{post.author.handle}"))
+ scrape_item.uploaded_at = date = self.parse_iso_date(post.record["createdAt"])
+ scrape_item.append_folders(self.create_separate_post_title(None, post.id, date))
+ self.create_eager_task(self.write_metadata(scrape_item, f"post {post.id}", post))
+
+ for media in _extract_media(post):
+ self.create_eager_task(self._media(scrape_item, media, post.author.did))
+ scrape_item.add_children()
+
+ async def _media(self, scrape_item: ScrapeItem, media: Media, did: str) -> None:
+ src = self.api.get_blob(did, media.cid)
+ with self.catch_errors(src):
+ name, ext = self.get_filename_and_ext(media.name, mime_type=media.mime)
+ await self.handle_file(
+ src,
+ scrape_item,
+ name,
+ ext,
+ custom_filename=name,
+ metadata=media,
+ )
+
+
+def _extract_media(post: PostView) -> Generator[Media]:
+ embed = post.record.get("embed")
+ if not embed:
+ return
+
+ embed["type"] = embed["$type"]
+ embed = cast("MediaAsset", embed) # pyright: ignore[reportInvalidCast]
+
+ match embed["type"]:
+ case "app.bsky.embed.images":
+ for img in embed["images"]:
+ yield parse_asset(img, "image")
+ case "app.bsky.embed.gallery":
+ for item in embed["items"]:
+ yield parse_asset(item, "image" if "image" in item else "video")
+ case "app.bsky.embed.image":
+ yield parse_asset(embed, "image")
+ case "app.bsky.embed.video":
+ yield parse_asset(embed, "video")
+ case "app.bsky.embed.record" | "app.bsky.embed.recordWithMedia":
+ # TODO: handle this
+ pass
+ case _:
+ raise ValueError(embed)
+
+
+def parse_asset(asset: Image | Video, key: Literal["image", "video"]) -> Media:
+ # TODO: handle video captions
+ res = Resolution(**ratio) if (ratio := asset.get("aspectRatio")) else None
+ blob: Blob = asset[key] # pyright: ignore[reportGeneralTypeIssues]
+ cid = blob["ref"]["$link"]
+ return Media(
+ type=key,
+ cid=cid,
+ name=asset.get("alt") or cid,
+ mime=blob["mimeType"],
+ resolution=res,
+ )
diff --git a/cyberdrop_dl/crawlers/bluesky/types.py b/cyberdrop_dl/crawlers/bluesky/types.py
new file mode 100644
index 000000000..a570d7157
--- /dev/null
+++ b/cyberdrop_dl/crawlers/bluesky/types.py
@@ -0,0 +1,119 @@
+"https://github.com/bluesky-social/atproto/blob/727701085829daf8e0440253346b5e70fc3810ad/lexicons/app/bsky/feed/defs.json"
+
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING, Any, Literal, NotRequired, Self, TypedDict
+
+from cyberdrop_dl.models import type_adapter
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.mediaprops import Resolution
+
+type FeedFilter = Literal[
+ "posts_with_replies",
+ "posts_no_replies",
+ "posts_with_media",
+ "posts_and_author_threads",
+ "posts_with_video",
+]
+
+
[email protected](frozen=True, slots=True)
+class ProfileViewBasic:
+ did: str
+ handle: str
+ display_name: str | None = None
+ created_at: str | None = None
+
+
[email protected](frozen=True, slots=True)
+class PostView:
+ uri: str
+ cid: str
+ author: ProfileViewBasic
+ record: Record # This is typed as unknown on their spec
+
+ @classmethod
+ def parse(cls, data: dict[str, Any]) -> Self:
+ return type_adapter(cls).validate_python(data)
+
+ @property
+ def id(self) -> str:
+ return self.uri.rpartition("/")[-1]
+
+ @property
+ def web_path(self) -> str:
+ return f"profile/{self.author.handle}/post/{self.id}"
+
+
+Href = TypedDict("Href", {"$link": str})
+Asset = TypedDict("Asset", {"$type": str})
+
+
+class Blob(Asset):
+ ref: Href
+ mimeType: str # noqa: N815
+ size: int
+
+
+class AspectRatio(TypedDict):
+ width: int
+ height: int
+
+
+class Image(Asset):
+ type: Literal["app.bsky.embed.image"]
+ image: Blob
+ alt: str
+ aspectRatio: AspectRatio | None # noqa: N815
+
+
+class Images(Asset):
+ type: Literal["app.bsky.embed.images"]
+ images: list[Image]
+
+
+class Gallery(Asset):
+ type: Literal["app.bsky.embed.gallery"]
+ items: list[Image]
+
+
+class Captions(TypedDict):
+ lang: str
+ file: Blob
+
+
+class Video(Asset):
+ type: Literal["app.bsky.embed.video"]
+ video: Blob
+ alt: str | None
+ aspectRatio: AspectRatio | None # noqa: N815
+ captions: Captions | None
+
+
+class External(Asset):
+ type: Literal["app.bsky.embed.external"]
+ uri: str
+ title: str
+ description: str
+ thumb: Blob
+ created_at: str | None
+ updated_at: str | None
+
+
+type MediaAsset = Video | Image | External | Images | Gallery
+
+
+class Record(TypedDict):
+ createdAt: str
+ embed: NotRequired[dict[str, Any]]
+
+
[email protected](frozen=True, slots=True)
+class Media:
+ type: Literal["video", "image"]
+ cid: str
+ name: str
+ mime: str
+ resolution: Resolution | None = None
diff --git a/tests/crawlers/test_cases/bluesky.py b/tests/crawlers/test_cases/bluesky.py
new file mode 100644
index 000000000..646506834
--- /dev/null
+++ b/tests/crawlers/test_cases/bluesky.py
@@ -0,0 +1,42 @@
+DOMAIN = "bluesky"
+TEST_CASES = [
+ {
+ "url": "https://bsky.app/profile/tenzilkemesq.bsky.social/post/3mt5h4og6jc2r",
+ "results": [
+ {
+ "url": "https://bsky.social/xrpc/com.atproto.sync.getBlob?did=did:plc:tbmq3lilsp2ukxhrha76cewi&cid=bafkreidlpgvgk5p5x5hrxwzot2lzy2gwfg2ggbujiuim5gg7adtxwiyeb4",
+ "filename": "Footage from Forza Horizon 6 of me breaking a hard 5000 xp bonus board..mp4",
+ "debrid_url": None,
+ "original_filename": "Footage from Forza Horizon 6 of me breaking a hard 5000 xp bonus board..mp4",
+ "referer": "https://bsky.app/profile/tenzilkemesq.bsky.social/post/3mt5h4og6jc2r",
+ "album_id": None,
+ "uploaded_at": 1786823521,
+ "download_folder": "re:@tenzilkemesq.bsky.social (BlueSky)/2026-08-15 - 3mt5h4og6jc2r",
+ "thumbnail": None,
+ }
+ ],
+ "count": 1,
+ },
+ {
+ "url": "https://bsky.app/profile/comicsguy67.bsky.social/post/3mupyd6tlyk2x",
+ "results": [
+ {
+ "url": "https://bsky.social/xrpc/com.atproto.sync.getBlob?did=did:plc:gtcw4ogmo7pegvqlcvuay4wv&cid=bafkreib27qhwltdaykcengqzs6k4i2w33u2uuh7ibadf3c5g4eda4owmou",
+ "filename": "bafkreib27qhwltdaykcengqzs6k4i2w33u2uuh7ibadf3c5g4eda4owmou.jpg",
+ "debrid_url": None,
+ "original_filename": "bafkreib27qhwltdaykcengqzs6k4i2w33u2uuh7ibadf3c5g4eda4owmou.jpg",
+ "referer": "https://bsky.app/profile/comicsguy67.bsky.social/post/3mupyd6tlyk2x",
+ "album_id": None,
+ "uploaded_at": 1788559980,
+ "download_folder": "re:@comicsguy67.bsky.social (BlueSky)/2026-09-04 - 3mupyd6tlyk2x",
+ "thumbnail": None,
+ }
+ ],
+ "count": 1,
+ },
+ {
+ "url": "https://bsky.app/profile/momo951.bsky.social",
+ "results": [],
+ "count": range(2, 100),
+ },
+]