Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add campaign support (Patreon) (#175)
NTFSvolumee21b3b2d7429
diff --git a/cyberdrop_dl/crawlers/patreon.py b/cyberdrop_dl/crawlers/patreon.pyindex bd10a6e31..de1e68e98 100644--- a/cyberdrop_dl/crawlers/patreon.py+++ b/cyberdrop_dl/crawlers/patreon.py@@ -1,172 +1,242 @@from __future__ import annotations-import asyncioimport dataclassesimport json-from typing import TYPE_CHECKING, Any, ClassVar+from typing import TYPE_CHECKING, Any, ClassVar, NotRequired, TypedDict, cast++from bs4 import BeautifulSoup+from typing_extensions import ReadOnlyfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.exceptions import NoExtensionError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper+from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, next_jsif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator-- from bs4 import BeautifulSoup+ from collections.abc import AsyncIterator, Generatorfrom cyberdrop_dl.url_objects import ScrapeItem@dataclasses.dataclass(slots=True)-class Asset:+class Media:+ id: strname: str | Noneurl: AbsoluteHttpURL- props: dict[str, Any]+ attributes: dict[str, Any]+++class Asset(TypedDict):+ id: str+ type: str+ attributes: ReadOnly[NotRequired[dict[str, Any]]]+ relationships: ReadOnly[NotRequired[dict[str, Any]]]+++class Included(Asset):+ attributes: dict[str, Any]+++class Post(Included):+ relationships: dict[str, Any]++ current_user_can_view: bool+ campaign_id: str+ published_at: str+ title: str+ url: strclass PatreonCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Post": "/posts/<slug>",+ "Creator": (+ "/<creator>",+ "/cw/<creator>",+ ),}DOMAIN: ClassVar[str] = "patreon"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.patreon.com")+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} - {title}"++ @property+ def separate_posts(self) -> bool:+ return Trueasync def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case ["posts", _]:return await self.post(scrape_item)+ case [creator] | ["cw", creator]:+ return await self.creator(scrape_item, creator)case _:raise ValueError@error_handling_wrapperasync def post(self, scrape_item: ScrapeItem) -> None:soup = await self.request_soup(scrape_item.url, impersonate=True)- bootstrap = _extract_bootstrap(soup)- post = _flatten_post(bootstrap["post"])- title = self.create_title(post["title"])+ post: dict[str, Any] = _extract_bootstrap(soup)["post"]+ self._post(+ scrape_item,+ post=_flatten_post(post["data"]),+ included=_flatten_included(post["included"]),+ )++ @error_handling_wrapper+ def _post(self, scrape_item: ScrapeItem, post: Post, included: dict[str, Included]):+ if not post["current_user_can_view"]:+ raise ScrapeError(402, "You do not have access to this post")++ campaign_name: str = included[post["campaign_id"]]["attributes"]["name"]+ title = self.create_title(campaign_name)scrape_item.setup_as_album(title)- scrape_item.uploaded_at = self.parse_iso_date(post["published_at"])- await self._post(scrape_item, post)-- async def _post(self, scrape_item: ScrapeItem, post: dict[str, Any]):- hashes: set[str] = set()-- def process_asset(item: ScrapeItem, asset: Asset) -> None:- hash = _md5(asset.url)- if hash:- if hash in hashes:- return- hashes.add(hash)- self.create_task(self._asset(item, asset))- scrape_item.add_children()- for name, get_assets in (("images", self._images), ("postfile", self._post_file)):- new_item = scrape_item.copy()- new_item.add_to_parent_title(name)- for asset in get_assets(post):- process_asset(new_item, asset)+ scrape_item.uploaded_at = date = self.parse_iso_date(post["published_at"])+ post_title = self.create_separate_post_title(post["title"], post["id"], date)+ scrape_item.add_to_parent_title(post_title)- new_item = scrape_item.copy()- new_item.add_to_parent_title("attachments")- async for asset in self._attachments(post):- process_asset(new_item, asset)+ self.create_task(self.write_metadata(scrape_item, f"post_{post['id']}", post))+ for media in self._parse_media(post, included):+ self.create_task(self._media(scrape_item, media))+ self.create_task(self.write_metadata(scrape_item, f"media_{media.id}", media))+ scrape_item.add_children()++ if embed := post.get("embed"):+ new_item = scrape_item.create_child(self.parse_url(embed["url"]))+ self.handle_embed(new_item)+ self.create_task(self.write_metadata(scrape_item, f"embed_{post['id']}", embed))+ scrape_item.add_children()@error_handling_wrapper- async def _asset(self, scrape_item: ScrapeItem, asset: Asset):- if asset.url.suffix == ".m3u8":- return await self._m3u8_asset(scrape_item, asset)+ async def _media(self, scrape_item: ScrapeItem, media: Media):+ if media.url.suffix == ".m3u8":+ return await self._m3u8_media(scrape_item, media)- name = asset.name+ name = media.nameif not name:- async with self.request(asset.url) as resp:+ async with self.request(media.url) as resp:name = resp.content_disposition.filename- filename, ext = self.get_filename_and_ext(name)- await self.handle_file(asset.url, scrape_item, filename, ext)+ try:+ filename, ext = self.get_filename_and_ext(name)+ except NoExtensionError:+ name = media.url.name+ filename, ext = self.get_filename_and_ext(name)++ await self.handle_file(media.url, scrape_item, name, ext, custom_filename=filename)- async def _m3u8_asset(self, scrape_item: ScrapeItem, asset: Asset):- m3u8, info = await self.request_m3u8_playlist(asset.url)+ async def _m3u8_media(self, scrape_item: ScrapeItem, media: Media):+ m3u8, info = await self.request_m3u8_playlist(media.url)filename = self.create_custom_filename(- asset.url.name.removesuffix(".m3u8"),+ media.url.name.removesuffix(".m3u8"),ext := ".mp4",resolution=info.resolution,video_codec=info.codecs.video,audio_codec=info.codecs.audio,)- await self.handle_file(asset.url, scrape_item, filename, ext, m3u8=m3u8)+ await self.handle_file(media.url, scrape_item, filename, ext, m3u8=m3u8)++ def _parse_media(self, post: Post, included: dict[str, Included]) -> Generator[Media]:+ media_ids: set[str] = set()+ if post_file := post.get("post_file"):+ media_id = str(post_file["media_id"])+ media_ids.add(media_id)+ url = self.parse_url(post_file["url"])+ yield Media(media_id, post_file.get("name"), url, post_file)++ for media_id in _get_post_media(post):+ media = included[media_id]+ attributes = media["attributes"]++ if media["type"] == "media" and (url := attributes.get("download_url")) and media_id not in media_ids:+ media_ids.add(media_id)+ yield Media(media_id, attributes.get("file_name"), self.parse_url(url), attributes)++ return+ # TODO: convert tiptap JSON to HTML or extract media ids from tiptap+ if not post["content"]:+ return+ soup = BeautifulSoup(post["content"], "html.parser")+ for media_id in css.iselect(soup, "[data-media-id]", "data-media-id"):+ if media_id in media_ids:+ continue+ self.log.warning("Found extra media id %s", media_id)+ media_ids.add(media_id)- def _images(self, post: dict[str, Any]) -> Generator[Asset]:- for image in post["images"]:- if url := image.get("download_url"):- yield Asset(image.get("file_name"), self.parse_url(url), image)-- def _post_file(self, post: dict[str, Any]) -> Generator[Asset]:- if postfile := post.get("post_file"):- url = self.parse_url(postfile["url"])- yield Asset(postfile.get("name"), url, postfile)-- async def _attachments(self, post: dict[str, Any]) -> AsyncGenerator[Asset]:- for fut in asyncio.as_completed(tuple(map(self._resolve_attachment, post["attachments"]))):- result = await fut- if result is not None:- yield result-- for attachment in post["attachments_media"]:- if url := attachment.get("download_url"):- yield Asset(attachment["file_name"], self.parse_url(url), attachment)+ @error_handling_wrapper+ async def creator(self, scrape_item: ScrapeItem, creator: str) -> None:+ campaign_id = await self._get_campaign_id(creator)+ await self.campaign(scrape_item, campaign_id)- async def _resolve_attachment(self, attachment: dict[str, str]) -> Asset | None:+ @error_handling_wrapper+ async def campaign(self, scrape_item: ScrapeItem, campaign_id: str) -> None:+ scrape_item.setup_as_profile("")+ api_url = (+ (self.PRIMARY_URL / "api/posts")+ .with_query(_CAMPAIGN_API_PARAMS)+ .extend_query(+ {+ "filter[campaign_id]": campaign_id,+ }+ )+ )+ async for resp in self._api_pager(api_url):+ included = _flatten_included(resp["included"])+ for post in resp["data"]:+ post = _flatten_post(post)+ new_item = scrape_item.create_child(self.parse_url(post["url"]))+ self._post(new_item, post, included)+ scrape_item.add_children()++ async def _api_pager(self, api_url: AbsoluteHttpURL) -> AsyncIterator[dict[str, Any]]:+ while True:+ resp = await self.request_json(api_url, impersonate=True)+ yield resp++ try:+ cursor = resp["meta"]["pagination"]["cursors"]["next"]+ except LookupError:+ break+ if not cursor:+ break+ api_url = api_url.update_query({"page[cursor]": cursor})++ async def _get_campaign_id(self, creator: str) -> str:+ soup = await self.request_soup(self.PRIMARY_URL / creator, impersonate=True)try:- url = await self._get_redirect_url(self.parse_url(attachment["url"]))- except Exception:- self.log.exception(f"Unable to resolve {attachment = }")+ bootstrap = _extract_bootstrap(soup)+ except css.SelectorError:+ # TODO: fix next_js chunk parsing+ return _extract_campaign_id(soup)else:- return Asset(attachment["name"], url, attachment)+ return bootstrap["campaign"]["data"]["id"]-def _extract_bootstrap(soup: BeautifulSoup) -> dict[str, Any]:- data = json.loads(css.select_text(soup, "#__NEXT_DATA__"))- envelope = data["props"]["pageProps"]["bootstrapEnvelope"]- return envelope.get("pageBootstrap") or envelope["bootstrap"]+def _extract_campaign_id(soup: BeautifulSoup):+ return get_text_between(str(soup), r"{\"value\":{\"campaign\":{\"data\":{\"id\":\"", r"\"")-def _flatten_included(included: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:- flatten = {}- for asset in included:- flatten.setdefault(asset["type"], {})[asset["id"]] = asset["attributes"]- return flatten+def _extract_campaign_id_next_js(soup: BeautifulSoup) -> str:+ included = next_js.ifind(next_js.extract(soup), "id", "type", "attributes")+ return next(incl["id"] for incl in included if incl["type"] == "campaign")-def _parse_post(post: dict[str, Any]) -> Generator[tuple[str, Any]]:- included = _flatten_included(post["included"])- post_data = post["data"]- relationships = post_data["relationships"]- campaign_id = relationships["campaign"]["data"]["id"]-- yield "id", int(post_data["id"])- yield from _parse_attributes(post_data["attributes"])+def _flatten_included(included: list[Included]) -> dict[str, Included]:+ return {incl["id"]: incl for incl in included}- yield "campaign", included["campaign"].pop(campaign_id)- yield from _parse_files(relationships, included)- yield "relationships", relationships- yield "included", included+def _extract_bootstrap(soup: BeautifulSoup) -> dict[str, Any]:+ data = json.loads(css.select_text(soup, "#__NEXT_DATA__"))+ envelope = data["props"]["pageProps"]["bootstrapEnvelope"]+ return envelope.get("pageBootstrap") or envelope["bootstrap"]-def _parse_files(- relationships: dict[str, dict[str, Any]], included: dict[str, dict[str, Any]]-) -> Generator[tuple[str, Any]]:- def extract_files(files: dict[str, list[dict[str, str]]]):- for file in files.get("data") or ():- yield included[file["type"]].pop(file["id"])- for key in (- "images",- "attachments",- "attachments_media",- ):- files = relationships.pop(key, {})- yield key, tuple(extract_files(files))+def _parse_post(post: dict[str, Any]) -> Generator[tuple[str, Any]]:+ yield "id", str(post["id"])+ yield from _parse_attributes(post["attributes"])+ yield "relationships", post["relationships"]+ yield "campaign_id", post["relationships"]["campaign"]["data"]["id"]def _parse_attributes(attributes: dict[str, Any]) -> Generator[tuple[str, Any]]:@@ -174,21 +244,114 @@ def _parse_attributes(attributes: dict[str, Any]) -> Generator[tuple[str, Any]]:json_keys = tuple(key for key in attributes if key.endswith(json_string))for json_key in json_keys:- value = attributes.pop(json_key.removesuffix(json_string), None)+ name = json_key.removesuffix(json_string)+ value = attributes.pop(name, None)json_value = attributes.pop(json_key, None)+ # TODO: convert to htmlif not value and json_value:value = json.loads(json_value)- yield json_key, value+ yield name, valueyield from attributes.items()-def _flatten_post(post: dict[str, Any]) -> dict[str, Any]:- return dict(sorted(_parse_post(post)))---def _md5(url: AbsoluteHttpURL) -> str | None:- for part in reversed(url.parts):- if len(part) == 32:- return part+def _flatten_post(post: dict[str, Any]) -> Post:+ return cast("Post", dict(sorted(_parse_post(post)))) # pyright: ignore[reportInvalidCast]+++def _get_post_media(post: Post) -> Generator[str]:+ for name in ("media", "video", "audio", "images", "attachments", "attachments_media"):+ relationships = post["relationships"].get(name, {}).get("data")+ if not relationships:+ continue++ if type(relationships) is not list:+ relationships = [relationships]++ for asset in relationships:+ yield asset["id"]+++_CAMPAIGN_API_PARAMS = (+ (+ "include",+ ",".join(+ (+ "campaign",+ "attachments",+ "attachments_media",+ "audio",+ "video",+ "images",+ "media",+ "native_video_insights",+ "user",+ )+ ),+ ),+ (+ "fields[campaign]",+ ",".join(+ (+ "currency",+ "show_audio_post_download_links",+ "avatar_photo_url",+ "avatar_photo_image_urls",+ "is_nsfw",+ "is_monthly",+ "name",+ "url",+ )+ ),+ ),+ (+ "fields[post]",+ ",".join(+ (+ "content",+ "content_json_string",+ "current_user_can_view",+ "embed",+ "image",+ "is_paid",+ "meta_image_url",+ "post_file",+ "post_metadata",+ "published_at",+ "post_type",+ "thumbnail",+ "thumbnail_url",+ "teaser_text",+ "title",+ "url",+ "moderation_status",+ )+ ),+ ),+ (+ "fields[user]",+ ",".join(+ (+ "image_url",+ "full_name",+ "url",+ )+ ),+ ),+ (+ "fields[media]",+ ",".join(+ (+ "id",+ "image_urls",+ "download_url",+ "metadata",+ "file_name",+ )+ ),+ ),+ ("sort", "-published_at"),+ ("filter[is_draft]", "false"),+ ("json-api-version", "1.0"),+)diff --git a/tests/crawlers/test_cases/patreon.py b/tests/crawlers/test_cases/patreon.pyindex c020a6af1..c1e3240f4 100644--- a/tests/crawlers/test_cases/patreon.py+++ b/tests/crawlers/test_cases/patreon.py@@ -11,7 +11,7 @@ TEST_CASES = ["referer": "https://www.patreon.com/posts/jujutsu-kaisen-1-150611726","album_id": None,"uploaded_at": 1770946069,- "download_folder": "re:Jujutsu Kaisen 3x7- -Tokyo Colony No. 1- -- Reaction Portion (Patreon)/images",+ "download_folder": "re:AutoSave (Patreon)/2026-02-12 - Jujutsu Kaisen 3x7- -Tokyo Colony No. 1- -- Rea",},{"url": "re:https://stream.mux.com/DsTORxvBjFq2mt2QRJ1cVNOROMjeg4OYpATQxxl5yWU.m3u8",@@ -21,7 +21,7 @@ TEST_CASES = ["referer": "https://www.patreon.com/posts/jujutsu-kaisen-1-150611726","album_id": None,"uploaded_at": 1770946069,- "download_folder": "re:Jujutsu Kaisen 3x7- -Tokyo Colony No. 1- -- Reaction Portion (Patreon)/postfile",+ "download_folder": "re:AutoSave (Patreon)/2026-02-12 - Jujutsu Kaisen 3x7- -Tokyo Colony No. 1- -- Rea",},],2,