Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add PeerTube support (#2116)
NTFSvolumec49d8d1aaf97
diff --git a/cyberdrop_dl/config/crawlers.py b/cyberdrop_dl/config/crawlers.pyindex cbd8855c1..b204b4271 100644--- a/cyberdrop_dl/config/crawlers.py+++ b/cyberdrop_dl/config/crawlers.py@@ -161,6 +161,7 @@ class GenericCrawlers(ConfigModel):chevereto: tuple[HttpURL, ...] = ()kvs: tuple[HttpURL, ...] = ()video: tuple[HttpURL, ...] = ()+ peertube: tuple[HttpURL, ...] = ()class Crawlers(ConfigGroup, name=None):diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yamlindex 9fa98eba4..ed91dfe34 100644--- a/cyberdrop_dl/config/default.yaml+++ b/cyberdrop_dl/config/default.yaml@@ -36,6 +36,7 @@ crawlers:chevereto: []discourse: []kvs: []+ peertube: []video: []wordpress_html: []wordpress_media: []diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.jsonindex 78c036dba..835034e25 100644--- a/cyberdrop_dl/config/schema.json+++ b/cyberdrop_dl/config/schema.json@@ -162,38 +162,38 @@"bandcamp": {"$ref": "#/$defs/BandcampConfig"},+ "clonr": {+ "$ref": "#/$defs/ClonrConfig"+ },"clypit": {"$ref": "#/$defs/ClypitConfig"},"generic": {"$ref": "#/$defs/GenericCrawlers"},- "one_pace": {- "$ref": "#/$defs/OnePaceConfig"- },- "tiktok": {- "$ref": "#/$defs/TikTokConfig"+ "google_drive": {+ "$ref": "#/$defs/GoogleDriveConfig"},- "twitter": {- "$ref": "#/$defs/TwitterConfig"+ "octave_music": {+ "$ref": "#/$defs/OctaveMusicConfig"},- "pawchive": {- "$ref": "#/$defs/KemonoConfig"+ "one_pace": {+ "$ref": "#/$defs/OnePaceConfig"},"only_haven": {"$ref": "#/$defs/KemonoConfig"},- "octave_music": {- "$ref": "#/$defs/OctaveMusicConfig"- },- "clonr": {- "$ref": "#/$defs/ClonrConfig"+ "pawchive": {+ "$ref": "#/$defs/KemonoConfig"},"pornhub": {"$ref": "#/$defs/PornHubConfig"},- "google_drive": {- "$ref": "#/$defs/GoogleDriveConfig"+ "tiktok": {+ "$ref": "#/$defs/TikTokConfig"+ },+ "twitter": {+ "$ref": "#/$defs/TwitterConfig"}},"title": "Crawlers",@@ -538,6 +538,14 @@},"title": "Video","type": "array"+ },+ "peertube": {+ "default": [],+ "items": {+ "$ref": "#/$defs/HttpURL"+ },+ "title": "Peertube",+ "type": "array"}},"title": "GenericCrawlers",@@ -556,6 +564,21 @@"GoogleDriveFormats": {"additionalProperties": false,"properties": {+ "docs": {+ "default": "docx",+ "enum": [+ "docx",+ "odt",+ "rtf",+ "txt",+ "epub",+ "pdf",+ "md",+ "zip"+ ],+ "title": "Docs",+ "type": "string"+ },"sheets": {"default": "xslx","enum": [@@ -576,21 +599,6 @@],"title": "Slides","type": "string"- },- "docs": {- "default": "docx",- "enum": [- "docx",- "odt",- "rtf",- "txt",- "epub",- "pdf",- "md",- "zip"- ],- "title": "Docs",- "type": "string"}},"title": "GoogleDriveFormats",diff --git a/cyberdrop_dl/crawlers/_peertube.py b/cyberdrop_dl/crawlers/_peertube.pynew file mode 100644index 000000000..f0c19a88a--- /dev/null+++ b/cyberdrop_dl/crawlers/_peertube.py@@ -0,0 +1,248 @@+from __future__ import annotations++import itertools+from typing import TYPE_CHECKING, Any, ClassVar, Self++from pydantic import dataclasses++from cyberdrop_dl import aio+from cyberdrop_dl.cache import disk_cached_method+from cyberdrop_dl.clients.http import HTTPConfig+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedDomains, SupportedPaths+from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.mediaprops import ISO639Subtitle, Resolution+from cyberdrop_dl.url_objects import AbsoluteHttpURL, MuxVideo, ScrapeItem+from cyberdrop_dl.utils import unique+from cyberdrop_dl.utils.dataclass import deserialize+from cyberdrop_dl.utils.errors import error_handling_wrapper++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator+++@HTTPConfig(rate_limit=(5, 1))[email protected]_path_builder("url")+class PeerTubeGenericCrawler(Crawler, is_generic=True):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Video": (+ "/w/<video_uuid>",+ "/w/<short_uuid>",+ "/videos/watch/<video_uuid>",+ "/videos/watch/<short_uuid>",+ ),+ "Playlist": "/w/p/<playlist_uuid>",+ "Account": "/a/<username>/videos",+ "Channel": "/c/<channel_uuid>",+ }+ DOMAIN: ClassVar[str] = "peertube"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://joinpeertube.org")++ def __init_subclass__(cls, **kwargs: Any) -> None:+ super().__init_subclass__(**kwargs)+ cls.DOMAIN = PeerTubeGenericCrawler.DOMAIN # pyright: ignore[reportConstantRedefinition]++ def __post_init__(self) -> None:+ self.api: PeerTubeAPI = PeerTubeAPI.from_crawler(self)+ self.instance_locks: aio.WeakAsyncLocks[AbsoluteHttpURL] = aio.WeakAsyncLocks()+ self.good_hosts: set[AbsoluteHttpURL] = set()+ self.bad_hosts: set[AbsoluteHttpURL] = set()++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["w", video_id] | ["videos", "watch", video_id]:+ await self.video(scrape_item, video_id)+ case ["a", playlist_id, "videos"]:+ await self.playlist(scrape_item, playlist_id, "accounts")+ case ["c", playlist_id]:+ await self.playlist(scrape_item, playlist_id, "video-channels")+ case ["w", "p", playlist_id]:+ await self.playlist(scrape_item, playlist_id, "video-playlists")+ case _:+ raise ValueError++ async def _check_node_info(self) -> None:+ if self.origin in self.good_hosts:+ return+ if self.origin in self.bad_hosts:+ raise NodeInfoError++ async with self.instance_locks[self.origin]:+ if self.origin in self.good_hosts:+ return+ if self.origin in self.bad_hosts:+ raise NodeInfoError++ node_info = await self.api.node_info()+ self.log.info("Server software running on %s:\n %s", self.origin, node_info["software"])+ self.good_hosts.add(self.origin)++ async def get_instances(self) -> tuple[str, ...]:+ try:+ return await self.api.instances()+ except Exception:+ self.log.exception("Unable to fetch additional PeerTube instances")+ return ()++ @error_handling_wrapper+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:+ if await self.check_complete_from_referer(scrape_item.url):+ return++ await self._check_node_info()+ video = await self.api.video(video_id, scrape_item.password)+ await self._video(scrape_item, video)++ async def _video(self, scrape_item: ScrapeItem, video: Video) -> None:+ scrape_item.uploaded_at = self.parse_iso_date(video.publishedAt)+ mux_streams = None+ best_video = max(f for f in video.files if f.hasVideo)+ if not best_video.hasAudio:+ best_audio = max((f for f in video.files if f.hasAudio and not f.hasVideo), key=lambda x: x.size)+ mux_streams = MuxVideo(best_video.fileUrl, best_audio.fileUrl)++ _, ext = self.get_filename_and_ext(best_video.fileUrl.name)+ filename = self.create_custom_filename(+ video.name, ext, file_id=video.uuid, resolution=best_video.resolution, fps=best_video.fps+ )+ subs = await self.api.captions(video.uuid, scrape_item.password)+ self.handle_subs(scrape_item, filename, subs)+ await self.handle_file(+ best_video.fileUrl,+ scrape_item,+ video.name,+ ext,+ custom_filename=filename,+ thumbnail=video.thumb,+ headers=self.api.headers(scrape_item.password),+ m3u8=mux_streams,+ )++ @error_handling_wrapper+ async def playlist(self, scrape_item: ScrapeItem, playlist_id: str, kind: str) -> None:+ info = await self.api.playlist(playlist_id, kind)+ self.create_eager_task(self.write_metadata(scrape_item, playlist_id, info))+ scrape_item.setup_as_album(self.create_title(info["displayName"], playlist_id), album_id=playlist_id)++ async for video in self.api.playlist_videos(playlist_id, kind):+ self.create_task(self.run(scrape_item.create_child(video)))+ scrape_item.add_children()+++class NodeInfoError(ScrapeError):+ def __init__(self, msg: str = "Unable to get nodeinfo from PeerTube instance") -> None:+ super().__init__("PeerTube Nodeinfo Error", msg)+++@HTTPConfig(headers={"Accept": "application/json"})+class PeerTubeAPI(API):+ @classmethod+ def headers(cls, password: str | None) -> dict[str, Any]:+ return {"x-peertube-video-password": password} if password else {}++ async def pager(self, url: AbsoluteHttpURL, count: int = 30) -> AsyncGenerator[list[dict[str, Any]]]:+ for start in itertools.count(0, count):+ resp = await self.request_json(url.update_query(start=start, count=count))+ yield resp["data"]+ if len(resp["data"]) < count:+ break++ @disk_cached_method(ttl=30 * 86400)+ async def instances(self) -> tuple[str, ...]:+ self.log.info("Fetching list of PeerTube instances")+ url = AbsoluteHttpURL("https://instances.joinpeertube.org/api/v1/instances?healthy=true")+ hosts: set[str] = set()+ async for data in self.pager(url, count=1_000):+ hosts.update(inst["host"] for inst in data)++ return tuple(sorted(hosts))++ async def video(self, video_id: str, password: str | None) -> Video:+ url = self.origin / "api/v1/videos" / video_id+ resp = await self.request_json(url, headers=self.headers(password))+ return Video.parse(resp)++ async def captions(self, video_id: str, password: str | None) -> list[ISO639Subtitle]:+ url = self.origin / "api/v1/videos" / video_id / "captions"+ resp = await self.request_json(url, headers=self.headers(password))+ return [ISO639Subtitle(sub["fileUrl"], sub["language"]["id"], sub["language"]["label"]) for sub in resp["data"]]++ async def playlist(self, playlist_id: str, kind: str) -> dict[str, Any]:+ url = self.origin / "api/v1" / kind / playlist_id+ return await self.request_json(url)++ async def playlist_videos(self, playlist_id: str, kind: str) -> AsyncGenerator[AbsoluteHttpURL]:+ url = self.origin / "api/v1" / kind / playlist_id / "videos"+ async for data in self.pager(url.with_query(sort="-publishedAt", nsfw="both", isLive="false")):+ for video in data:+ uuid: str = video.get("shortUUID") or video["video"]["shortUUID"]+ yield self.origin / "w" / uuid++ async def node_info(self) -> dict[str, Any]:+ # https://nodeinfo.diaspora.software/protocol.html+ url = self.origin / ".well-known/nodeinfo"++ try:+ resp = await self.request_json(url)+ except Exception as e:+ raise NodeInfoError from e++ try:+ for link in resp["links"]:+ rel: str = link["rel"]+ if rel.startswith("http://nodeinfo.diaspora.software/ns/schema"):+ return await self.request_json(self.parse_url(link["href"]))++ except Exception as e:+ raise NodeInfoError("PeerTube instance does not support the nodeinfo protocol") from e++ raise NodeInfoError("PeerTube instance does not support the nodeinfo protocol")++[email protected](slots=True, frozen=True)+class Video:+ uuid: str+ name: str+ url: AbsoluteHttpURL+ thumb: AbsoluteHttpURL+ publishedAt: str+ files: tuple[File, ...]++ @classmethod+ def parse(cls, video: dict[str, Any]) -> Self:+ if video.get("isLive"):+ raise ScrapeError(422, "Livestreams are not supported")++ *_, thumb = max((t["width"], t["height"], t["fileUrl"]) for t in video["thumbnails"])++ files = unique(+ itertools.chain(+ itertools.chain.from_iterable(p["files"] for p in video["streamingPlaylists"]),+ video["files"],+ ),+ key=lambda x: x["id"],+ )+ return deserialize(cls, video, thumb=thumb, files=map(File.parse, files))+++# ruff: noqa: N815[email protected](slots=True, frozen=True, order=True)+class File:+ resolution: Resolution | None+ fps: int | None+ size: int+ id: int+ fileUrl: AbsoluteHttpURL+ hasAudio: bool+ hasVideo: bool++ @classmethod+ def parse(cls, file: dict[str, Any]) -> Self:+ return deserialize(+ cls,+ file,+ resolution=Resolution(file["width"], file["height"]) if file.get("hasVideo") else None,+ )+++class PeerTubeCrawler(PeerTubeGenericCrawler):+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ("peertube",)diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex 84c64700f..5200be938 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -4,6 +4,7 @@ import asyncioimport contextlibimport dataclassesimport logging+import sysimport timefrom pathlib import Pathfrom typing import TYPE_CHECKING, Literal, Self, override@@ -29,7 +30,7 @@ from cyberdrop_dl.utils import remove_trailing_slashfrom cyberdrop_dl.utils._url import matches_any_hostif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator, Iterable, Iterator+ from collections.abc import AsyncGenerator, Generator, Iteratorfrom cyberdrop_dl.clients.jd.client import JDownloaderfrom cyberdrop_dl.config import Config@@ -43,10 +44,6 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)-def _filter_by_domain(url: AbsoluteHttpURL, domains: Iterable[str]) -> bool:- return any(domain in url.host for domain in domains)--@dataclasses.dataclass(slots=True, eq=False)class CrawlerFactory:manager: Manager@@ -59,6 +56,12 @@ class CrawlerFactory:return f"<{type(self).__name__}(instances={len(self._instances):,})>"def __getitem__[CrawlerT: Crawler](self, obj: type[CrawlerT]) -> CrawlerT:+ instance = self.get(obj)+ if instance is None:+ raise KeyError(obj)+ return instance++ def __call__[CrawlerT: Crawler](self, obj: type[CrawlerT]) -> CrawlerT:instance = self.get(obj)if instance is None:instance = self._instances[obj] = obj(self.manager, self.task_mngr, self.tui)@@ -149,7 +152,7 @@ class ScrapeMapper:self.tui.scrape.get_queue = self._scrape_queueself.tui.downloads.get_queue = self._download_queue- def _init_crawlers(self) -> None:+ async def _init_crawlers(self) -> None:crawlers = get_crawlers_mapping()self.crawlers.update(crawlers)@@ -162,6 +165,33 @@ class ScrapeMapper:logger.debug(msg)_disable_crawlers_by_config(self.crawlers, *self.manager.config.crawlers.disabled)+ await self._register_peertube()++ async def _register_peertube(self) -> None:+ # User may have disabled peertube+ if "peertube" not in self.crawlers:+ return++ if "pytest" in sys.modules:+ return++ from cyberdrop_dl.crawlers._peertube import PeerTubeCrawler++ for url in self.manager.config.crawlers.generic.peertube:+ if other := _best_match(self.crawlers, url.host):+ msg = GENERIC_MAP_ERROR.format(url, PeerTubeCrawler.NAME, other.NAME)+ logger.error(msg)+ continue++ self.crawlers[url.host] = PeerTubeCrawler++ peertube = self._factory(PeerTubeCrawler)+ for host in await peertube.get_instances():+ crawler = self.crawlers.setdefault(host, PeerTubeCrawler)+ if crawler.DOMAIN == PeerTubeCrawler.DOMAIN:+ continue++ logger.warning("Found PeerTube site '%s' mapped to a non PeerTube crawler: %s", host, crawler.INFO)@contextlib.asynccontextmanagerasync def __call__(self) -> AsyncGenerator[Self]:@@ -217,7 +247,7 @@ class ScrapeMapper:async def __async_init__(self) -> None:if self._ready:return- self._init_crawlers()+ await self._init_crawlers()try:await self._jdownloader.connect(self.manager.http_client)except Exception:@@ -265,7 +295,7 @@ class ScrapeMapper:async def _send_to_crawler(self, scrape_item: ScrapeItem) -> None:if cls := _best_match(self.crawlers, scrape_item.url.host):- crawler = self._factory[cls]+ crawler = self._factory(cls)await crawler.__async_init__()if crawler.__url_config__.trim:scrape_item.url = remove_trailing_slash(scrape_item.url)@@ -330,6 +360,13 @@ def get_crawlers_mapping() -> dict[str, type[Crawler]]:return crawlers_map+GENERIC_MAP_ERROR = (+ "Unable to assign {} to generic crawler {}. "+ "URL conflicts with URL format of builtin crawler {}. "+ "URL will be ignored"+)++def register_crawler(crawlers_map: dict[str, type[Crawler]],crawler: type[Crawler],@@ -343,11 +380,7 @@ def register_crawler(if not other and (match := _best_match(crawlers_map, crawler.PRIMARY_URL.host)):other = matchif other:- msg = (- f"Unable to assign {crawler.PRIMARY_URL} to generic crawler {crawler.NAME}. "- f"URL conflicts with URL format of builtin crawler {other.NAME}. "- "URL will be ignored"- )+ msg = GENERIC_MAP_ERROR.format(crawler.PRIMARY_URL, crawler.NAME, other.NAME)if from_user == "raise":raise ValueError(msg)logger.error(msg)diff --git a/tests/crawlers/test_cases/peertube.py b/tests/crawlers/test_cases/peertube.pynew file mode 100644index 000000000..f230dcc4c--- /dev/null+++ b/tests/crawlers/test_cases/peertube.py@@ -0,0 +1,52 @@+DOMAIN = "peertube"+TEST_CASES = [+ {+ "url": "https://framatube.org/w/217eefeb-883d-45be-b7fc-a788ad8507d3",+ "results": [+ {+ "url": "https://framatube.org/lazy-static/video-captions/217eefeb-883d-45be-b7fc-a788ad8507d3-fr.vtt",+ "filename": "What is PeerTube (english subtitles) [217eefeb-883d-45be-b7fc-a788ad8507d3][1080p@24fps].fr.vtt",+ "debrid_url": None,+ "original_filename": "French",+ "referer": "https://framatube.org/w/217eefeb-883d-45be-b7fc-a788ad8507d3#What%20is%20PeerTube%20(english%20subtitles)%20%5B217eefeb-883d-45be-b7fc-a788ad8507d3%5D%5B1080p@24fps%5D.fr.vtt",+ "album_id": None,+ "uploaded_at": 1527168124,+ "download_folder": "re:Loose Files (Framatube.org)",+ "thumbnail": None,+ },+ {+ "url": "https://framatube.org/lazy-static/video-captions/217eefeb-883d-45be-b7fc-a788ad8507d3-de.vtt",+ "filename": "What is PeerTube (english subtitles) [217eefeb-883d-45be-b7fc-a788ad8507d3][1080p@24fps].de.vtt",+ "debrid_url": None,+ "original_filename": "German",+ "referer": "https://framatube.org/w/217eefeb-883d-45be-b7fc-a788ad8507d3#What%20is%20PeerTube%20(english%20subtitles)%20%5B217eefeb-883d-45be-b7fc-a788ad8507d3%5D%5B1080p@24fps%5D.de.vtt",+ "album_id": None,+ "uploaded_at": 1527168124,+ "download_folder": "re:Loose Files (Framatube.org)",+ "thumbnail": None,+ },+ {+ "url": "https://framatube.org/lazy-static/video-captions/217eefeb-883d-45be-b7fc-a788ad8507d3-en.vtt",+ "filename": "What is PeerTube (english subtitles) [217eefeb-883d-45be-b7fc-a788ad8507d3][1080p@24fps].en.vtt",+ "debrid_url": None,+ "original_filename": "English",+ "referer": "https://framatube.org/w/217eefeb-883d-45be-b7fc-a788ad8507d3#What%20is%20PeerTube%20(english%20subtitles)%20%5B217eefeb-883d-45be-b7fc-a788ad8507d3%5D%5B1080p@24fps%5D.en.vtt",+ "album_id": None,+ "uploaded_at": 1527168124,+ "download_folder": "re:Loose Files (Framatube.org)",+ "thumbnail": None,+ },+ {+ "url": "https://framatube.org/static/streaming-playlists/hls/217eefeb-883d-45be-b7fc-a788ad8507d3/a109ecd1-f2f5-4da7-a17e-808d5e5e9ec8-1080-fragmented.mp4",+ "filename": "What is PeerTube (english subtitles) [217eefeb-883d-45be-b7fc-a788ad8507d3][1080p@24fps].mp4",+ "debrid_url": None,+ "original_filename": "What is PeerTube? (english subtitles)",+ "referer": "https://framatube.org/w/217eefeb-883d-45be-b7fc-a788ad8507d3",+ "album_id": None,+ "uploaded_at": 1527168124,+ "download_folder": "re:Loose Files (Framatube.org)",+ "thumbnail": "https://framatube.org/lazy-static/thumbnails/8f59e65b-89a1-4105-a5bd-4183b252efcf.jpg",+ },+ ],+ },+]diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.pyindex 1555112cf..a37267869 100644--- a/tests/crawlers/test_crawlers.py+++ b/tests/crawlers/test_crawlers.py@@ -60,7 +60,7 @@ async def test_crawler(running_manager: Manager, test_case: test_cases.CrawlerTeNone,)assert cls, f"{test_case.domain} is not a valid crawler domain. Test case is invalid"- crawler = scrape_mapper._factory[cls]+ crawler = scrape_mapper._factory(cls)await crawler.__async_init__()item = ScrapeItem.from_url(crawler.parse_url(test_case.url))item.download_folder = running_manager.config.download_folder