Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: extract URLs in post (BlueSky)

NTFSvolume89aec5c86192
diff --git a/cyberdrop_dl/crawlers/bluesky/crawler.py b/cyberdrop_dl/crawlers/bluesky/crawler.py
index a74790722..5c48658f6 100644
--- a/cyberdrop_dl/crawlers/bluesky/crawler.py
+++ b/cyberdrop_dl/crawlers/bluesky/crawler.py
@@ -73,6 +73,10 @@ class BlueskyCrawler(Crawler):
self.create_eager_task(self._media(scrape_item, media, post.author.did))
scrape_item.add_children()
+ for url in _extract_links(post.record):
+ self.handle_external_links(scrape_item.create_child(self.parse_url(url)))
+ scrape_item.add_children()
+
async def _media(self, scrape_item: ScrapeItem, media: Media, did: str) -> None:
src = await self.api.get_blob(did, media.cid)
with self.catch_errors(src):
@@ -106,6 +110,13 @@ def _extract_media(record: Mapping[str, Any]) -> Generator[Media]:
)
+def _extract_links(record: Mapping[str, Any]) -> Generator[str]:
+ for facet in record.get("facets", ()):
+ for feat in facet["features"]:
+ if feat["$type"] == "app.bsky.richtext.facet#link":
+ yield feat["uri"]
+
+
def _normalize_blob(blob: Blob | LegacyBlob) -> Blob:
# https://atproto.com/specs/data-model#blob-type
# https://atproto.com/specs/data-model#usage-and-implementation-guidelines
diff --git a/cyberdrop_dl/crawlers/bluesky/types.py b/cyberdrop_dl/crawlers/bluesky/types.py
index 9c8b1cdfd..91254c3b4 100644
--- a/cyberdrop_dl/crawlers/bluesky/types.py
+++ b/cyberdrop_dl/crawlers/bluesky/types.py
@@ -3,7 +3,7 @@
from __future__ import annotations
import dataclasses
-from typing import TYPE_CHECKING, Any, Literal, NotRequired, Self, TypedDict
+from typing import TYPE_CHECKING, Any, Literal, Self, TypedDict
from cyberdrop_dl.models import type_adapter
@@ -32,7 +32,7 @@ class PostView:
uri: str
cid: str
author: ProfileViewBasic
- record: Record # This is typed as unknown on their spec
+ record: dict[str, Any] # This is typed as unknown on their spec
@classmethod
def parse(cls, data: dict[str, Any]) -> Self:
@@ -110,11 +110,6 @@ class External(Asset):
type MediaAsset = Video | Image | Images | Gallery | External
-class Record(TypedDict):
- createdAt: str
- embed: NotRequired[dict[str, Any]]
-
-
@dataclasses.dataclass(frozen=True, slots=True)
class Media:
type: str