Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: parse media from TIPTAP JSON (Patreon) (#2184)
NTFSvolume0cc707d55ba1
diff --git a/cyberdrop_dl/crawlers/patreon.py b/cyberdrop_dl/crawlers/patreon.pyindex 8bb560dc2..0969fc7d9 100644--- a/cyberdrop_dl/crawlers/patreon.py+++ b/cyberdrop_dl/crawlers/patreon.py@@ -3,15 +3,16 @@ from __future__ import annotationsimport contextlibimport dataclassesimport json-from typing import TYPE_CHECKING, Any, ClassVar, NotRequired, TypedDict, cast+from typing import TYPE_CHECKING, Any, ClassVar, NotRequired, TypedDict, TypeGuard, castfrom typing_extensions import ReadOnly+from cyberdrop_dl import aiofrom cyberdrop_dl.clients.http import HTTPConfigfrom cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import NoExtensionError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, extr_text, next_js+from cyberdrop_dl.utils import css, extr_text, next_js, traversalfrom cyberdrop_dl.utils.dataclass import DictDataclassfrom cyberdrop_dl.utils.errors import error_handling_wrapper@@ -101,14 +102,14 @@ class PatreonCrawler(Crawler):@error_handling_wrapperasync def post(self, scrape_item: ScrapeItem, post_id: str) -> None:post = await self.api.post(post_id)- self._post(+ await self._post(scrape_item,post=_flatten_post(post["data"]),included=_flatten_included(post["included"]),)@error_handling_wrapper- def _post(self, scrape_item: ScrapeItem, post: Post, included: dict[str, Included]) -> None:+ async def _post(self, scrape_item: ScrapeItem, post: Post, included: dict[str, Included]) -> None:if not post["current_user_can_view"]:raise ScrapeError(402, "You do not have access to this post")@@ -119,9 +120,10 @@ class PatreonCrawler(Crawler):scrape_item.uploaded_at = date = self.parse_iso_date(post["published_at"])post_title = self.create_separate_post_title(post["title"], post["id"], date)scrape_item.append_folders(post_title)-self.create_eager_task(self.write_metadata(scrape_item, f"post_{post['id']}", post))+ await self._post_media(scrape_item, post, included)+ async def _post_media(self, scrape_item: ScrapeItem, post: Post, included: dict[str, Included]) -> None:medias = tuple(_parse_media(post, included))unique_medias = tuple(_unique_media(medias))self.log.debug(@@ -132,20 +134,56 @@ class PatreonCrawler(Crawler):LazyLogMedia(medias),)- for media in unique_medias:- self.create_task(self._media(scrape_item, media))- self.create_eager_task(self.write_metadata(scrape_item, f"media_{media.id}", media))- scrape_item.add_children()+ seen_urls: set[AbsoluteHttpURL] = set()+ seen_media_ids: set[str] = {m.id for m in unique_medias}+ sleep = aio.periodic_sleep(10)- if embed := post.get("embed"):- self.log.debug("Found embed in post %s: %s", post["id"], embed)- new_item = scrape_item.create_child(self.parse_url(embed["url"]))- self.handle_embed(new_item)- self.create_eager_task(self.write_metadata(scrape_item, f"embed_{post['id']}", embed))+ async def dispatch_url(link: str, *, embed: bool) -> None:+ url = self.parse_url(link)+ if url in seen_urls or url.host == self.PRIMARY_URL.host:+ return++ new_item = scrape_item.create_child(url)+ seen_urls.add(new_item.url)+ self.handle_external_links(new_item, reset=not embed)scrape_item.add_children()+ await sleep()++ async with self.new_task_group() as tg:+ for media in unique_medias:+ tg.create_task(self._media(scrape_item, media))+ scrape_item.add_children()+ await sleep()++ if embed := post.get("embed"):+ self.log.debug("Found embed in post %s: %s", post["id"], embed)+ tg.create_task(self.write_metadata(scrape_item, f"embed_{post['id']}", embed))+ await dispatch_url(embed["url"], embed=True)++ links, assets = _extract_json_tiptap_media(post)+ if not (links or assets):+ return++ self.log.debug("Found embeded TIPTAP media in post %s: %s", post["id"], {"links": links, "assets": assets})++ for url in links:+ await dispatch_url(url, embed=False)++ for asset in assets:+ if asset["id"] not in seen_media_ids:+ seen_media_ids.add(asset["id"])+ tg.create_task(self.media(scrape_item, asset["id"]))+ scrape_item.add_children()+ await sleep()++ @error_handling_wrapper+ async def media(self, scrape_item: ScrapeItem, media_id: str) -> None:+ media = await self.api.media(media_id)+ await self._media(scrape_item, media)@error_handling_wrapperasync def _media(self, scrape_item: ScrapeItem, media: Media) -> None:+ self.create_eager_task(self.write_metadata(scrape_item, f"media_{media.id}", media))if media.url.suffix == ".m3u8":return await self._m3u8_media(scrape_item, media)@@ -191,13 +229,16 @@ class PatreonCrawler(Crawler):@error_handling_wrapperasync def campaign(self, scrape_item: ScrapeItem, campaign_id: str) -> None:scrape_item.setup_as_profile("")- async for resp in self.api.posts(campaign_id):- included = _flatten_included(resp["included"])- for post in resp["data"]:- post = _flatten_post(post)- new_item = scrape_item.create_child(self.parse_url(post["url"]))- self._post(new_item, post, included)- scrape_item.add_children()+ sleep = aio.periodic_sleep(10)+ async with self.new_task_group() as tg:+ async for resp in self.api.posts(campaign_id):+ included = _flatten_included(resp["included"])+ for post in resp["data"]:+ post = _flatten_post(post)+ new_item = scrape_item.create_child(self.parse_url(post["url"]))+ tg.create_task(self._post(new_item, post, included))+ scrape_item.add_children()+ await sleep()class LazyLogMedia:@@ -207,7 +248,7 @@ class LazyLogMedia:def __json__(self) -> dict[str, dict[str, Any]]:return {m.id: m.__json__() for m in self.media}- def __str__(self):+ def __str__(self) -> str:return str(self.__json__())@@ -221,6 +262,15 @@ class PatreonV1API(API):url = (self.ENTRYPOINT / "posts" / post_id).with_query(_API_PARAMS)return await self.request_json(url)+ async def media(self, media_id: str) -> Media:+ url = (self.ENTRYPOINT / "media" / media_id).with_query(_API_PARAMS)+ resp = await self.request_json(url)+ attrs = resp["data"]["attributes"]+ if not (url := attrs.get("download_url")):+ raise ScrapeError(403, f"No download URL found for {media_id}")++ return Media(media_id, attrs.get("file_name"), self.parse_url(url), attrs)+async def campaign_id(self, creator: str) -> str:async with self.request(self.PRIMARY_URL / creator, impersonate=True) as resp:soup = await resp.soup()@@ -291,20 +341,29 @@ def _parse_post(post: dict[str, Any]) -> Generator[tuple[str, Any]]:def _parse_attributes(attributes: dict[str, Any]) -> Generator[tuple[str, Any]]:- json_string = "_json_string"- json_keys = tuple(key for key in attributes if key.endswith(json_string))-- for json_key in json_keys:- name = json_key.removesuffix(json_string)- value = attributes.pop(name, None)- json_value = attributes.pop(json_key, None)- # TODO: convert to html- if not value and json_value:- value = json.loads(json_value)+ for name, value in attributes.items():+ if value and name.endswith("_json_string"):+ value = json.loads(value)yield name, value- yield from attributes.items()++def _extract_json_tiptap_media(post: Post) -> tuple[list[str], list[Included]]:+ links: list[str] = []+ assets: list[Included] = []+ for name, value in post.items():+ if value and name.endswith("_json_string"):+ for _, elem in traversal.traverse(value, _tiptap_media_visitor):+ attrs = elem["attrs"]+ match media_type := elem["type"]:+ case "link" if href := attrs.get("href"):+ links.append(href)+ case "audio" | "video" | "image" if media_id := attrs.get("media_id"):+ assets.append({"id": media_id, "type": media_type, "attributes": attrs})+ case _:+ continue++ return links, assetsdef _flatten_post(post: dict[str, Any]) -> Post:@@ -363,7 +422,29 @@ def _parse_media(post: Post, included: dict[str, Included]) -> Generator[Media]:if m_hash := _md5_from_url(url):yield Media(m_hash, None, url, image)- # TODO: convert tiptap JSON to HTML or extract media ids from tiptap++def _tiptap_media_visitor(_: object, obj: object) -> TypeGuard[dict[str, Any]]:+ if not (type(obj) is dict and (t_type := obj.get("type"))):+ return False++ if t_type in {"audio", "link", "video", "image"}:+ return bool(obj.get("attrs"))++ if t_type not in {+ "hardBreak",+ "horizontalRule",+ "listItem",+ "bulletList",+ "orderedList",+ "blockquote",+ "paragraph",+ "text",+ "doc",+ "bold",+ }:+ PatreonCrawler.get_logger().warning("Unknown TIPTAP element: %s", obj)++ return False_API_PARAMS = (@@ -402,23 +483,24 @@ _API_PARAMS = ("fields[post]",",".join((- "content","content_json_string",+ "content","current_user_can_view","embed","image","is_paid","meta_image_url",+ "moderation_status","post_file","post_metadata",- "published_at","post_type",- "thumbnail",- "thumbnail_url",+ "published_at",+ "teaser_text_json_string","teaser_text",+ "thumbnail_url",+ "thumbnail","title","url",- "moderation_status",)),),