Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: extract URLs in post (BlueSky)
NTFSvolume89aec5c86192
diff --git a/cyberdrop_dl/crawlers/bluesky/crawler.py b/cyberdrop_dl/crawlers/bluesky/crawler.pyindex a74790722..5c48658f6 100644--- a/cyberdrop_dl/crawlers/bluesky/crawler.py+++ b/cyberdrop_dl/crawlers/bluesky/crawler.py@@ -73,6 +73,10 @@ class BlueskyCrawler(Crawler):self.create_eager_task(self._media(scrape_item, media, post.author.did))scrape_item.add_children()+ for url in _extract_links(post.record):+ self.handle_external_links(scrape_item.create_child(self.parse_url(url)))+ scrape_item.add_children()+async def _media(self, scrape_item: ScrapeItem, media: Media, did: str) -> None:src = await self.api.get_blob(did, media.cid)with self.catch_errors(src):@@ -106,6 +110,13 @@ def _extract_media(record: Mapping[str, Any]) -> Generator[Media]:)+def _extract_links(record: Mapping[str, Any]) -> Generator[str]:+ for facet in record.get("facets", ()):+ for feat in facet["features"]:+ if feat["$type"] == "app.bsky.richtext.facet#link":+ yield feat["uri"]++def _normalize_blob(blob: Blob | LegacyBlob) -> Blob:# https://atproto.com/specs/data-model#blob-type# https://atproto.com/specs/data-model#usage-and-implementation-guidelinesdiff --git a/cyberdrop_dl/crawlers/bluesky/types.py b/cyberdrop_dl/crawlers/bluesky/types.pyindex 9c8b1cdfd..91254c3b4 100644--- a/cyberdrop_dl/crawlers/bluesky/types.py+++ b/cyberdrop_dl/crawlers/bluesky/types.py@@ -3,7 +3,7 @@from __future__ import annotationsimport dataclasses-from typing import TYPE_CHECKING, Any, Literal, NotRequired, Self, TypedDict+from typing import TYPE_CHECKING, Any, Literal, Self, TypedDictfrom cyberdrop_dl.models import type_adapter@@ -32,7 +32,7 @@ class PostView:uri: strcid: strauthor: ProfileViewBasic- record: Record # This is typed as unknown on their spec+ record: dict[str, Any] # This is typed as unknown on their spec@classmethoddef parse(cls, data: dict[str, Any]) -> Self:@@ -110,11 +110,6 @@ class External(Asset):type MediaAsset = Video | Image | Images | Gallery | External-class Record(TypedDict):- createdAt: str- embed: NotRequired[dict[str, Any]]--@dataclasses.dataclass(frozen=True, slots=True)class Media:type: str