Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: parse media from TIPTAP JSON (Patreon) (#2184)

NTFSvolume0cc707d55ba1
diff --git a/cyberdrop_dl/crawlers/patreon.py b/cyberdrop_dl/crawlers/patreon.py
index 8bb560dc2..0969fc7d9 100644
--- a/cyberdrop_dl/crawlers/patreon.py
+++ b/cyberdrop_dl/crawlers/patreon.py
@@ -3,15 +3,16 @@ from __future__ import annotations
import contextlib
import dataclasses
import json
-from typing import TYPE_CHECKING, Any, ClassVar, NotRequired, TypedDict, cast
+from typing import TYPE_CHECKING, Any, ClassVar, NotRequired, TypedDict, TypeGuard, cast
from typing_extensions import ReadOnly
+from cyberdrop_dl import aio
from cyberdrop_dl.clients.http import HTTPConfig
from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths
from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, extr_text, next_js
+from cyberdrop_dl.utils import css, extr_text, next_js, traversal
from cyberdrop_dl.utils.dataclass import DictDataclass
from cyberdrop_dl.utils.errors import error_handling_wrapper
@@ -101,14 +102,14 @@ class PatreonCrawler(Crawler):
@error_handling_wrapper
async def post(self, scrape_item: ScrapeItem, post_id: str) -> None:
post = await self.api.post(post_id)
- self._post(
+ await self._post(
scrape_item,
post=_flatten_post(post["data"]),
included=_flatten_included(post["included"]),
)
@error_handling_wrapper
- def _post(self, scrape_item: ScrapeItem, post: Post, included: dict[str, Included]) -> None:
+ async def _post(self, scrape_item: ScrapeItem, post: Post, included: dict[str, Included]) -> None:
if not post["current_user_can_view"]:
raise ScrapeError(402, "You do not have access to this post")
@@ -119,9 +120,10 @@ class PatreonCrawler(Crawler):
scrape_item.uploaded_at = date = self.parse_iso_date(post["published_at"])
post_title = self.create_separate_post_title(post["title"], post["id"], date)
scrape_item.append_folders(post_title)
-
self.create_eager_task(self.write_metadata(scrape_item, f"post_{post['id']}", post))
+ await self._post_media(scrape_item, post, included)
+ async def _post_media(self, scrape_item: ScrapeItem, post: Post, included: dict[str, Included]) -> None:
medias = tuple(_parse_media(post, included))
unique_medias = tuple(_unique_media(medias))
self.log.debug(
@@ -132,20 +134,56 @@ class PatreonCrawler(Crawler):
LazyLogMedia(medias),
)
- for media in unique_medias:
- self.create_task(self._media(scrape_item, media))
- self.create_eager_task(self.write_metadata(scrape_item, f"media_{media.id}", media))
- scrape_item.add_children()
+ seen_urls: set[AbsoluteHttpURL] = set()
+ seen_media_ids: set[str] = {m.id for m in unique_medias}
+ sleep = aio.periodic_sleep(10)
- if embed := post.get("embed"):
- self.log.debug("Found embed in post %s: %s", post["id"], embed)
- new_item = scrape_item.create_child(self.parse_url(embed["url"]))
- self.handle_embed(new_item)
- self.create_eager_task(self.write_metadata(scrape_item, f"embed_{post['id']}", embed))
+ async def dispatch_url(link: str, *, embed: bool) -> None:
+ url = self.parse_url(link)
+ if url in seen_urls or url.host == self.PRIMARY_URL.host:
+ return
+
+ new_item = scrape_item.create_child(url)
+ seen_urls.add(new_item.url)
+ self.handle_external_links(new_item, reset=not embed)
scrape_item.add_children()
+ await sleep()
+
+ async with self.new_task_group() as tg:
+ for media in unique_medias:
+ tg.create_task(self._media(scrape_item, media))
+ scrape_item.add_children()
+ await sleep()
+
+ if embed := post.get("embed"):
+ self.log.debug("Found embed in post %s: %s", post["id"], embed)
+ tg.create_task(self.write_metadata(scrape_item, f"embed_{post['id']}", embed))
+ await dispatch_url(embed["url"], embed=True)
+
+ links, assets = _extract_json_tiptap_media(post)
+ if not (links or assets):
+ return
+
+ self.log.debug("Found embeded TIPTAP media in post %s: %s", post["id"], {"links": links, "assets": assets})
+
+ for url in links:
+ await dispatch_url(url, embed=False)
+
+ for asset in assets:
+ if asset["id"] not in seen_media_ids:
+ seen_media_ids.add(asset["id"])
+ tg.create_task(self.media(scrape_item, asset["id"]))
+ scrape_item.add_children()
+ await sleep()
+
+ @error_handling_wrapper
+ async def media(self, scrape_item: ScrapeItem, media_id: str) -> None:
+ media = await self.api.media(media_id)
+ await self._media(scrape_item, media)
@error_handling_wrapper
async def _media(self, scrape_item: ScrapeItem, media: Media) -> None:
+ self.create_eager_task(self.write_metadata(scrape_item, f"media_{media.id}", media))
if media.url.suffix == ".m3u8":
return await self._m3u8_media(scrape_item, media)
@@ -191,13 +229,16 @@ class PatreonCrawler(Crawler):
@error_handling_wrapper
async def campaign(self, scrape_item: ScrapeItem, campaign_id: str) -> None:
scrape_item.setup_as_profile("")
- async for resp in self.api.posts(campaign_id):
- included = _flatten_included(resp["included"])
- for post in resp["data"]:
- post = _flatten_post(post)
- new_item = scrape_item.create_child(self.parse_url(post["url"]))
- self._post(new_item, post, included)
- scrape_item.add_children()
+ sleep = aio.periodic_sleep(10)
+ async with self.new_task_group() as tg:
+ async for resp in self.api.posts(campaign_id):
+ included = _flatten_included(resp["included"])
+ for post in resp["data"]:
+ post = _flatten_post(post)
+ new_item = scrape_item.create_child(self.parse_url(post["url"]))
+ tg.create_task(self._post(new_item, post, included))
+ scrape_item.add_children()
+ await sleep()
class LazyLogMedia:
@@ -207,7 +248,7 @@ class LazyLogMedia:
def __json__(self) -> dict[str, dict[str, Any]]:
return {m.id: m.__json__() for m in self.media}
- def __str__(self):
+ def __str__(self) -> str:
return str(self.__json__())
@@ -221,6 +262,15 @@ class PatreonV1API(API):
url = (self.ENTRYPOINT / "posts" / post_id).with_query(_API_PARAMS)
return await self.request_json(url)
+ async def media(self, media_id: str) -> Media:
+ url = (self.ENTRYPOINT / "media" / media_id).with_query(_API_PARAMS)
+ resp = await self.request_json(url)
+ attrs = resp["data"]["attributes"]
+ if not (url := attrs.get("download_url")):
+ raise ScrapeError(403, f"No download URL found for {media_id}")
+
+ return Media(media_id, attrs.get("file_name"), self.parse_url(url), attrs)
+
async def campaign_id(self, creator: str) -> str:
async with self.request(self.PRIMARY_URL / creator, impersonate=True) as resp:
soup = await resp.soup()
@@ -291,20 +341,29 @@ def _parse_post(post: dict[str, Any]) -> Generator[tuple[str, Any]]:
def _parse_attributes(attributes: dict[str, Any]) -> Generator[tuple[str, Any]]:
- json_string = "_json_string"
- json_keys = tuple(key for key in attributes if key.endswith(json_string))
-
- for json_key in json_keys:
- name = json_key.removesuffix(json_string)
- value = attributes.pop(name, None)
- json_value = attributes.pop(json_key, None)
- # TODO: convert to html
- if not value and json_value:
- value = json.loads(json_value)
+ for name, value in attributes.items():
+ if value and name.endswith("_json_string"):
+ value = json.loads(value)
yield name, value
- yield from attributes.items()
+
+def _extract_json_tiptap_media(post: Post) -> tuple[list[str], list[Included]]:
+ links: list[str] = []
+ assets: list[Included] = []
+ for name, value in post.items():
+ if value and name.endswith("_json_string"):
+ for _, elem in traversal.traverse(value, _tiptap_media_visitor):
+ attrs = elem["attrs"]
+ match media_type := elem["type"]:
+ case "link" if href := attrs.get("href"):
+ links.append(href)
+ case "audio" | "video" | "image" if media_id := attrs.get("media_id"):
+ assets.append({"id": media_id, "type": media_type, "attributes": attrs})
+ case _:
+ continue
+
+ return links, assets
def _flatten_post(post: dict[str, Any]) -> Post:
@@ -363,7 +422,29 @@ def _parse_media(post: Post, included: dict[str, Included]) -> Generator[Media]:
if m_hash := _md5_from_url(url):
yield Media(m_hash, None, url, image)
- # TODO: convert tiptap JSON to HTML or extract media ids from tiptap
+
+def _tiptap_media_visitor(_: object, obj: object) -> TypeGuard[dict[str, Any]]:
+ if not (type(obj) is dict and (t_type := obj.get("type"))):
+ return False
+
+ if t_type in {"audio", "link", "video", "image"}:
+ return bool(obj.get("attrs"))
+
+ if t_type not in {
+ "hardBreak",
+ "horizontalRule",
+ "listItem",
+ "bulletList",
+ "orderedList",
+ "blockquote",
+ "paragraph",
+ "text",
+ "doc",
+ "bold",
+ }:
+ PatreonCrawler.get_logger().warning("Unknown TIPTAP element: %s", obj)
+
+ return False
_API_PARAMS = (
@@ -402,23 +483,24 @@ _API_PARAMS = (
"fields[post]",
",".join(
(
- "content",
"content_json_string",
+ "content",
"current_user_can_view",
"embed",
"image",
"is_paid",
"meta_image_url",
+ "moderation_status",
"post_file",
"post_metadata",
- "published_at",
"post_type",
- "thumbnail",
- "thumbnail_url",
+ "published_at",
+ "teaser_text_json_string",
"teaser_text",
+ "thumbnail_url",
+ "thumbnail",
"title",
"url",
- "moderation_status",
)
),
),