Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: request all post media (Patreon)
NTFSvolume08a2000f3480
diff --git a/cyberdrop_dl/crawlers/patreon.py b/cyberdrop_dl/crawlers/patreon.pyindex ce7a5024b..8bb560dc2 100644--- a/cyberdrop_dl/crawlers/patreon.py+++ b/cyberdrop_dl/crawlers/patreon.py@@ -12,22 +12,36 @@ from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import NoExtensionError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css, extr_text, next_js+from cyberdrop_dl.utils.dataclass import DictDataclassfrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator+ from collections.abc import AsyncGenerator, Generator, Iterablefrom bs4 import BeautifulSoupfrom cyberdrop_dl.url_objects import ScrapeItem[email protected](slots=True)[email protected](frozen=True, slots=True, order=True)class Media:id: strname: str | Noneurl: AbsoluteHttpURLattributes: dict[str, Any]+ hash: str | None = None++ __iter__ = DictDataclass.__iter__++ def __json__(self) -> dict[str, Any]:+ me = dict(self)+ del me["attributes"]+ me["url"] = str(self.url)+ return me++ def __post_init__(self) -> None:+ if not self.hash:+ object.__setattr__(self, "hash", _md5_from_url(self.url))class Asset(TypedDict):@@ -107,12 +121,24 @@ class PatreonCrawler(Crawler):scrape_item.append_folders(post_title)self.create_eager_task(self.write_metadata(scrape_item, f"post_{post['id']}", post))- for media in self._parse_media(post, included):++ medias = tuple(_parse_media(post, included))+ unique_medias = tuple(_unique_media(medias))+ self.log.debug(+ "Found %s media assets in post %s (%s unique)\n%s",+ len(medias),+ post["id"],+ len(unique_medias),+ LazyLogMedia(medias),+ )++ for media in unique_medias:self.create_task(self._media(scrape_item, media))self.create_eager_task(self.write_metadata(scrape_item, f"media_{media.id}", media))scrape_item.add_children()if embed := post.get("embed"):+ self.log.debug("Found embed in post %s: %s", post["id"], embed)new_item = scrape_item.create_child(self.parse_url(embed["url"]))self.handle_embed(new_item)self.create_eager_task(self.write_metadata(scrape_item, f"embed_{post['id']}", embed))@@ -126,7 +152,10 @@ class PatreonCrawler(Crawler):name = media.nameif not name:async with self.request(media.url) as resp:- name = resp.content_disposition.filename+ try:+ name = resp.content_disposition.filename+ except ScrapeError:+ name = media.url.name or media.url.parent.nametry:filename, ext = self.get_filename_and_ext(name)@@ -134,47 +163,26 @@ class PatreonCrawler(Crawler):name = media.url.namefilename, ext = self.get_filename_and_ext(name)- await self.handle_file(media.url, scrape_item, name, ext, custom_filename=filename)+ await self.handle_file(+ media.url,+ scrape_item,+ name,+ ext,+ custom_filename=self.create_custom_filename(filename, ext, file_id=media.id),+ )async def _m3u8_media(self, scrape_item: ScrapeItem, media: Media) -> None:m3u8, info = await self.request_m3u8_playlist(media.url)filename = self.create_custom_filename(- media.url.name.removesuffix(".m3u8"),+ media.hash or media.url.name.removesuffix(".m3u8"),ext := ".mp4",+ file_id=media.id,resolution=info.resolution,video_codec=info.codecs.video,audio_codec=info.codecs.audio,)await self.handle_file(media.url, scrape_item, filename, ext, m3u8=m3u8)- def _parse_media(self, post: Post, included: dict[str, Included]) -> Generator[Media]:- media_ids: set[str] = set()- if post_file := post.get("post_file"):- media_id = str(post_file["media_id"])- media_ids.add(media_id)- url = self.parse_url(post_file["url"])- yield Media(media_id, post_file.get("name"), url, post_file)-- if image := post.get("image"):- url = self.parse_url(image.get("large_url") or image["url"])- try:- media_id = url.parts[url.parts.index(post["id"]) + 1]- except ValueError:- pass- else:- media_ids.add(media_id)- yield Media(media_id, None, url, image)-- for media_id in _get_post_media(post):- media = included[media_id]- attributes = media["attributes"]-- if media["type"] == "media" and (url := attributes.get("download_url")) and media_id not in media_ids:- media_ids.add(media_id)- yield Media(media_id, attributes.get("file_name"), self.parse_url(url), attributes)-- # TODO: convert tiptap JSON to HTML or extract media ids from tiptap-@error_handling_wrapperasync def creator(self, scrape_item: ScrapeItem, creator: str) -> None:campaign_id = await self.api.campaign_id(creator)@@ -192,6 +200,17 @@ class PatreonCrawler(Crawler):scrape_item.add_children()+class LazyLogMedia:+ def __init__(self, media: tuple[Media, ...]) -> None:+ self.media: tuple[Media, ...] = media++ def __json__(self) -> dict[str, dict[str, Any]]:+ return {m.id: m.__json__() for m in self.media}++ def __str__(self):+ return str(self.__json__())++@HTTPConfig(impersonate=True)class PatreonV1API(API):# Public API v1 is retiring on October 7, 2026.@@ -199,7 +218,7 @@ class PatreonV1API(API):ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.patreon.com/api")async def post(self, post_id: str) -> dict[str, Any]:- url = (self.ENTRYPOINT / "posts" / post_id).with_query({"json-api-version": 1.0})+ url = (self.ENTRYPOINT / "posts" / post_id).with_query(_API_PARAMS)return await self.request_json(url)async def campaign_id(self, creator: str) -> str:@@ -219,7 +238,7 @@ class PatreonV1API(API):def posts(self, campaign_id: str) -> AsyncGenerator[dict[str, Any]]:url = ((self.ENTRYPOINT / "posts")- .with_query(_CAMPAIGN_API_PARAMS)+ .with_query(_API_PARAMS).extend_query({"filter[campaign_id]": campaign_id,@@ -305,7 +324,49 @@ def _get_post_media(post: Post) -> Generator[str]:yield asset["id"]-_CAMPAIGN_API_PARAMS = (+def _md5_from_url(url: AbsoluteHttpURL) -> str | None:+ for p in reversed(url.parts):+ if len(p) == 32:+ return p+++def _unique_media(medias: Iterable[Media]) -> Generator[Media]:+ media_ids: set[str] = set()+ media_hashes: set[str] = set()++ for media in medias:+ if media.id in media_ids:+ continue+ media_ids.add(media.id)+ if media.hash:+ if media.hash in media_hashes:+ continue+ media_hashes.add(media.hash)+ yield media+++def _parse_media(post: Post, included: dict[str, Included]) -> Generator[Media]:+ if post_file := post.get("post_file"):+ media_id = str(post_file["media_id"])+ url = PatreonCrawler.parse_url(post_file["url"])+ yield Media(media_id, post_file.get("name"), url, post_file)++ for media_id in _get_post_media(post):+ media = included[media_id]+ attributes = media["attributes"]++ if media["type"] == "media" and (url := attributes.get("download_url")):+ yield Media(media_id, attributes.get("file_name"), PatreonCrawler.parse_url(url), attributes)++ if image := post.get("image"):+ url = PatreonCrawler.parse_url(image.get("large_url") or image["url"])+ if m_hash := _md5_from_url(url):+ yield Media(m_hash, None, url, image)++ # TODO: convert tiptap JSON to HTML or extract media ids from tiptap+++_API_PARAMS = (("include",",".join(diff --git a/scripts/tools/make_test_cases.py b/scripts/tools/make_test_cases.pyindex ff3828766..ae84af7cf 100644--- a/scripts/tools/make_test_cases.py+++ b/scripts/tools/make_test_cases.py@@ -36,6 +36,8 @@ def parse_jsonl(file: Path) -> Generator[tuple[str, str, TestCase]]:base = Path.cwd() / Config().download_folderfor line in file.read_text(encoding="utf-8").splitlines():media = json.loads(line)+ if media["url"].startswith("metadata:"):+ continueurl = media["parents"][0] if media["parents"] else media["referer"]media["download_folder"] = "re:" + str(Path(media["download_folder"]).relative_to(base))yield media["domain"], url, {key: media[key] for key in KEYS}diff --git a/tests/crawlers/test_cases/patreon.py b/tests/crawlers/test_cases/patreon.pyindex b90d97863..24efe2416 100644--- a/tests/crawlers/test_cases/patreon.py+++ b/tests/crawlers/test_cases/patreon.py@@ -4,24 +4,61 @@ TEST_CASES = ["url": "https://www.patreon.com/posts/jujutsu-kaisen-1-150611726","results": [{- "url": "re:fe88322e8f1441acb3ee40ea26c71baa",- "filename": "jjk 3x7 copy.jpg",+ "url": "re:https://c10.patreonusercontent.com/4/patreon-media/p/post/150611726/817dda46b0b44ca19c65ccabebb804d1/eyJhIjoxLCJwIjoxfQ%3D%3D/1.mp4?token-hash=",+ "filename": "- video_preview [612978699].mp4",+ "debrid_url": None,+ "original_filename": " - video_preview.mp4",+ "referer": "https://www.patreon.com/posts/jujutsu-kaisen-1-150611726",+ "album_id": None,+ "uploaded_at": 1770946069,+ "download_folder": "re:AutoSave (Patreon)/2026-02-13 - Jujutsu Kaisen 3x7- -Tokyo Colony No. 1- -- Rea",+ "thumbnail": None,+ },+ {+ "url": "re:https://c10.patreonusercontent.com/4/patreon-media/p/post/150611726/fe88322e8f1441acb3ee40ea26c71baa/eyJhIjoxLCJwIjoxfQ%3D%3D/1.jpg?token-hash=",+ "filename": "jjk 3x7 copy [613013785].jpg","debrid_url": None,"original_filename": "jjk 3x7 copy.jpg","referer": "https://www.patreon.com/posts/jujutsu-kaisen-1-150611726","album_id": None,"uploaded_at": 1770946069,- "download_folder": "re:AutoSave (Patreon)/2026-02-13 - Jujutsu Kaisen 3x7- -Tokyo Colony No. 1",+ "download_folder": "re:AutoSave (Patreon)/2026-02-13 - Jujutsu Kaisen 3x7- -Tokyo Colony No. 1- -- Rea",+ "thumbnail": None,+ },+ {+ "url": "re:https://c10.patreonusercontent.com/4/patreon-media/p/media/612978699/f33a9e8f75614b3c810bbd174277906f/eyJhIjoxLCJwIjoxfQ%3D%3D/1.png?token-hash=",+ "filename": "1 [612978998].png",+ "debrid_url": None,+ "original_filename": "1.png",+ "referer": "https://www.patreon.com/posts/jujutsu-kaisen-1-150611726",+ "album_id": None,+ "uploaded_at": 1770946069,+ "download_folder": "re:AutoSave (Patreon)/2026-02-13 - Jujutsu Kaisen 3x7- -Tokyo Colony No. 1- -- Rea",+ "thumbnail": None,+ },+ {+ "url": "re:https://stream.mux.com/DsTORxvBjFq2mt2QRJ1cVNOROMjeg4OYpATQxxl5yWU.m3u8?token=",+ "filename": "DsTORxvBjFq2mt2QRJ1cVNOROMjeg4OYpATQxxl5yWU [612973032][avc1][mp4a][1080p].mp4",+ "debrid_url": None,+ "original_filename": "DsTORxvBjFq2mt2QRJ1cVNOROMjeg4OYpATQxxl5yWU [612973032][avc1][mp4a][1080p].mp4",+ "referer": "https://www.patreon.com/posts/jujutsu-kaisen-1-150611726",+ "album_id": None,+ "uploaded_at": 1770946069,+ "download_folder": "re:AutoSave (Patreon)/2026-02-13 - Jujutsu Kaisen 3x7- -Tokyo Colony No. 1- -- Rea",+ "thumbnail": None,},{- "url": "re:https://stream.mux.com/DsTORxvBjFq2mt2QRJ1cVNOROMjeg4OYpATQxxl5yWU.m3u8",- "filename": "DsTORxvBjFq2mt2QRJ1cVNOROMjeg4OYpATQxxl5yWU [avc1][mp4a][1080p].mp4",+ "url": "re:https://stream.mux.com/tJM4PVqsLiZOSCqZBPtM00U6eUqO3kIhPTw01eO8yELbQ/720p.mp4?token=",+ "filename": "720p [612978697].mp4","debrid_url": None,- "original_filename": "DsTORxvBjFq2mt2QRJ1cVNOROMjeg4OYpATQxxl5yWU [avc1][mp4a][1080p].mp4",+ "original_filename": "720p.mp4","referer": "https://www.patreon.com/posts/jujutsu-kaisen-1-150611726","album_id": None,"uploaded_at": 1770946069,+ "download_folder": "re:AutoSave (Patreon)/2026-02-13 - Jujutsu Kaisen 3x7- -Tokyo Colony No. 1- -- Rea",+ "thumbnail": None,},],+ "count": 5,}]