Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add PeerTube support (#2116)

NTFSvolumec49d8d1aaf97
diff --git a/cyberdrop_dl/config/crawlers.py b/cyberdrop_dl/config/crawlers.py
index cbd8855c1..b204b4271 100644
--- a/cyberdrop_dl/config/crawlers.py
+++ b/cyberdrop_dl/config/crawlers.py
@@ -161,6 +161,7 @@ class GenericCrawlers(ConfigModel):
chevereto: tuple[HttpURL, ...] = ()
kvs: tuple[HttpURL, ...] = ()
video: tuple[HttpURL, ...] = ()
+ peertube: tuple[HttpURL, ...] = ()
class Crawlers(ConfigGroup, name=None):
diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yaml
index 9fa98eba4..ed91dfe34 100644
--- a/cyberdrop_dl/config/default.yaml
+++ b/cyberdrop_dl/config/default.yaml
@@ -36,6 +36,7 @@ crawlers:
chevereto: []
discourse: []
kvs: []
+ peertube: []
video: []
wordpress_html: []
wordpress_media: []
diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.json
index 78c036dba..835034e25 100644
--- a/cyberdrop_dl/config/schema.json
+++ b/cyberdrop_dl/config/schema.json
@@ -162,38 +162,38 @@
"bandcamp": {
"$ref": "#/$defs/BandcampConfig"
},
+ "clonr": {
+ "$ref": "#/$defs/ClonrConfig"
+ },
"clypit": {
"$ref": "#/$defs/ClypitConfig"
},
"generic": {
"$ref": "#/$defs/GenericCrawlers"
},
- "one_pace": {
- "$ref": "#/$defs/OnePaceConfig"
- },
- "tiktok": {
- "$ref": "#/$defs/TikTokConfig"
+ "google_drive": {
+ "$ref": "#/$defs/GoogleDriveConfig"
},
- "twitter": {
- "$ref": "#/$defs/TwitterConfig"
+ "octave_music": {
+ "$ref": "#/$defs/OctaveMusicConfig"
},
- "pawchive": {
- "$ref": "#/$defs/KemonoConfig"
+ "one_pace": {
+ "$ref": "#/$defs/OnePaceConfig"
},
"only_haven": {
"$ref": "#/$defs/KemonoConfig"
},
- "octave_music": {
- "$ref": "#/$defs/OctaveMusicConfig"
- },
- "clonr": {
- "$ref": "#/$defs/ClonrConfig"
+ "pawchive": {
+ "$ref": "#/$defs/KemonoConfig"
},
"pornhub": {
"$ref": "#/$defs/PornHubConfig"
},
- "google_drive": {
- "$ref": "#/$defs/GoogleDriveConfig"
+ "tiktok": {
+ "$ref": "#/$defs/TikTokConfig"
+ },
+ "twitter": {
+ "$ref": "#/$defs/TwitterConfig"
}
},
"title": "Crawlers",
@@ -538,6 +538,14 @@
},
"title": "Video",
"type": "array"
+ },
+ "peertube": {
+ "default": [],
+ "items": {
+ "$ref": "#/$defs/HttpURL"
+ },
+ "title": "Peertube",
+ "type": "array"
}
},
"title": "GenericCrawlers",
@@ -556,6 +564,21 @@
"GoogleDriveFormats": {
"additionalProperties": false,
"properties": {
+ "docs": {
+ "default": "docx",
+ "enum": [
+ "docx",
+ "odt",
+ "rtf",
+ "txt",
+ "epub",
+ "pdf",
+ "md",
+ "zip"
+ ],
+ "title": "Docs",
+ "type": "string"
+ },
"sheets": {
"default": "xslx",
"enum": [
@@ -576,21 +599,6 @@
],
"title": "Slides",
"type": "string"
- },
- "docs": {
- "default": "docx",
- "enum": [
- "docx",
- "odt",
- "rtf",
- "txt",
- "epub",
- "pdf",
- "md",
- "zip"
- ],
- "title": "Docs",
- "type": "string"
}
},
"title": "GoogleDriveFormats",
diff --git a/cyberdrop_dl/crawlers/_peertube.py b/cyberdrop_dl/crawlers/_peertube.py
new file mode 100644
index 000000000..f0c19a88a
--- /dev/null
+++ b/cyberdrop_dl/crawlers/_peertube.py
@@ -0,0 +1,248 @@
+from __future__ import annotations
+
+import itertools
+from typing import TYPE_CHECKING, Any, ClassVar, Self
+
+from pydantic import dataclasses
+
+from cyberdrop_dl import aio
+from cyberdrop_dl.cache import disk_cached_method
+from cyberdrop_dl.clients.http import HTTPConfig
+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedDomains, SupportedPaths
+from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.mediaprops import ISO639Subtitle, Resolution
+from cyberdrop_dl.url_objects import AbsoluteHttpURL, MuxVideo, ScrapeItem
+from cyberdrop_dl.utils import unique
+from cyberdrop_dl.utils.dataclass import deserialize
+from cyberdrop_dl.utils.errors import error_handling_wrapper
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator
+
+
+@HTTPConfig(rate_limit=(5, 1))
[email protected]_path_builder("url")
+class PeerTubeGenericCrawler(Crawler, is_generic=True):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Video": (
+ "/w/<video_uuid>",
+ "/w/<short_uuid>",
+ "/videos/watch/<video_uuid>",
+ "/videos/watch/<short_uuid>",
+ ),
+ "Playlist": "/w/p/<playlist_uuid>",
+ "Account": "/a/<username>/videos",
+ "Channel": "/c/<channel_uuid>",
+ }
+ DOMAIN: ClassVar[str] = "peertube"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://joinpeertube.org")
+
+ def __init_subclass__(cls, **kwargs: Any) -> None:
+ super().__init_subclass__(**kwargs)
+ cls.DOMAIN = PeerTubeGenericCrawler.DOMAIN # pyright: ignore[reportConstantRedefinition]
+
+ def __post_init__(self) -> None:
+ self.api: PeerTubeAPI = PeerTubeAPI.from_crawler(self)
+ self.instance_locks: aio.WeakAsyncLocks[AbsoluteHttpURL] = aio.WeakAsyncLocks()
+ self.good_hosts: set[AbsoluteHttpURL] = set()
+ self.bad_hosts: set[AbsoluteHttpURL] = set()
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["w", video_id] | ["videos", "watch", video_id]:
+ await self.video(scrape_item, video_id)
+ case ["a", playlist_id, "videos"]:
+ await self.playlist(scrape_item, playlist_id, "accounts")
+ case ["c", playlist_id]:
+ await self.playlist(scrape_item, playlist_id, "video-channels")
+ case ["w", "p", playlist_id]:
+ await self.playlist(scrape_item, playlist_id, "video-playlists")
+ case _:
+ raise ValueError
+
+ async def _check_node_info(self) -> None:
+ if self.origin in self.good_hosts:
+ return
+ if self.origin in self.bad_hosts:
+ raise NodeInfoError
+
+ async with self.instance_locks[self.origin]:
+ if self.origin in self.good_hosts:
+ return
+ if self.origin in self.bad_hosts:
+ raise NodeInfoError
+
+ node_info = await self.api.node_info()
+ self.log.info("Server software running on %s:\n %s", self.origin, node_info["software"])
+ self.good_hosts.add(self.origin)
+
+ async def get_instances(self) -> tuple[str, ...]:
+ try:
+ return await self.api.instances()
+ except Exception:
+ self.log.exception("Unable to fetch additional PeerTube instances")
+ return ()
+
+ @error_handling_wrapper
+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
+ if await self.check_complete_from_referer(scrape_item.url):
+ return
+
+ await self._check_node_info()
+ video = await self.api.video(video_id, scrape_item.password)
+ await self._video(scrape_item, video)
+
+ async def _video(self, scrape_item: ScrapeItem, video: Video) -> None:
+ scrape_item.uploaded_at = self.parse_iso_date(video.publishedAt)
+ mux_streams = None
+ best_video = max(f for f in video.files if f.hasVideo)
+ if not best_video.hasAudio:
+ best_audio = max((f for f in video.files if f.hasAudio and not f.hasVideo), key=lambda x: x.size)
+ mux_streams = MuxVideo(best_video.fileUrl, best_audio.fileUrl)
+
+ _, ext = self.get_filename_and_ext(best_video.fileUrl.name)
+ filename = self.create_custom_filename(
+ video.name, ext, file_id=video.uuid, resolution=best_video.resolution, fps=best_video.fps
+ )
+ subs = await self.api.captions(video.uuid, scrape_item.password)
+ self.handle_subs(scrape_item, filename, subs)
+ await self.handle_file(
+ best_video.fileUrl,
+ scrape_item,
+ video.name,
+ ext,
+ custom_filename=filename,
+ thumbnail=video.thumb,
+ headers=self.api.headers(scrape_item.password),
+ m3u8=mux_streams,
+ )
+
+ @error_handling_wrapper
+ async def playlist(self, scrape_item: ScrapeItem, playlist_id: str, kind: str) -> None:
+ info = await self.api.playlist(playlist_id, kind)
+ self.create_eager_task(self.write_metadata(scrape_item, playlist_id, info))
+ scrape_item.setup_as_album(self.create_title(info["displayName"], playlist_id), album_id=playlist_id)
+
+ async for video in self.api.playlist_videos(playlist_id, kind):
+ self.create_task(self.run(scrape_item.create_child(video)))
+ scrape_item.add_children()
+
+
+class NodeInfoError(ScrapeError):
+ def __init__(self, msg: str = "Unable to get nodeinfo from PeerTube instance") -> None:
+ super().__init__("PeerTube Nodeinfo Error", msg)
+
+
+@HTTPConfig(headers={"Accept": "application/json"})
+class PeerTubeAPI(API):
+ @classmethod
+ def headers(cls, password: str | None) -> dict[str, Any]:
+ return {"x-peertube-video-password": password} if password else {}
+
+ async def pager(self, url: AbsoluteHttpURL, count: int = 30) -> AsyncGenerator[list[dict[str, Any]]]:
+ for start in itertools.count(0, count):
+ resp = await self.request_json(url.update_query(start=start, count=count))
+ yield resp["data"]
+ if len(resp["data"]) < count:
+ break
+
+ @disk_cached_method(ttl=30 * 86400)
+ async def instances(self) -> tuple[str, ...]:
+ self.log.info("Fetching list of PeerTube instances")
+ url = AbsoluteHttpURL("https://instances.joinpeertube.org/api/v1/instances?healthy=true")
+ hosts: set[str] = set()
+ async for data in self.pager(url, count=1_000):
+ hosts.update(inst["host"] for inst in data)
+
+ return tuple(sorted(hosts))
+
+ async def video(self, video_id: str, password: str | None) -> Video:
+ url = self.origin / "api/v1/videos" / video_id
+ resp = await self.request_json(url, headers=self.headers(password))
+ return Video.parse(resp)
+
+ async def captions(self, video_id: str, password: str | None) -> list[ISO639Subtitle]:
+ url = self.origin / "api/v1/videos" / video_id / "captions"
+ resp = await self.request_json(url, headers=self.headers(password))
+ return [ISO639Subtitle(sub["fileUrl"], sub["language"]["id"], sub["language"]["label"]) for sub in resp["data"]]
+
+ async def playlist(self, playlist_id: str, kind: str) -> dict[str, Any]:
+ url = self.origin / "api/v1" / kind / playlist_id
+ return await self.request_json(url)
+
+ async def playlist_videos(self, playlist_id: str, kind: str) -> AsyncGenerator[AbsoluteHttpURL]:
+ url = self.origin / "api/v1" / kind / playlist_id / "videos"
+ async for data in self.pager(url.with_query(sort="-publishedAt", nsfw="both", isLive="false")):
+ for video in data:
+ uuid: str = video.get("shortUUID") or video["video"]["shortUUID"]
+ yield self.origin / "w" / uuid
+
+ async def node_info(self) -> dict[str, Any]:
+ # https://nodeinfo.diaspora.software/protocol.html
+ url = self.origin / ".well-known/nodeinfo"
+
+ try:
+ resp = await self.request_json(url)
+ except Exception as e:
+ raise NodeInfoError from e
+
+ try:
+ for link in resp["links"]:
+ rel: str = link["rel"]
+ if rel.startswith("http://nodeinfo.diaspora.software/ns/schema"):
+ return await self.request_json(self.parse_url(link["href"]))
+
+ except Exception as e:
+ raise NodeInfoError("PeerTube instance does not support the nodeinfo protocol") from e
+
+ raise NodeInfoError("PeerTube instance does not support the nodeinfo protocol")
+
+
[email protected](slots=True, frozen=True)
+class Video:
+ uuid: str
+ name: str
+ url: AbsoluteHttpURL
+ thumb: AbsoluteHttpURL
+ publishedAt: str
+ files: tuple[File, ...]
+
+ @classmethod
+ def parse(cls, video: dict[str, Any]) -> Self:
+ if video.get("isLive"):
+ raise ScrapeError(422, "Livestreams are not supported")
+
+ *_, thumb = max((t["width"], t["height"], t["fileUrl"]) for t in video["thumbnails"])
+
+ files = unique(
+ itertools.chain(
+ itertools.chain.from_iterable(p["files"] for p in video["streamingPlaylists"]),
+ video["files"],
+ ),
+ key=lambda x: x["id"],
+ )
+ return deserialize(cls, video, thumb=thumb, files=map(File.parse, files))
+
+
+# ruff: noqa: N815
[email protected](slots=True, frozen=True, order=True)
+class File:
+ resolution: Resolution | None
+ fps: int | None
+ size: int
+ id: int
+ fileUrl: AbsoluteHttpURL
+ hasAudio: bool
+ hasVideo: bool
+
+ @classmethod
+ def parse(cls, file: dict[str, Any]) -> Self:
+ return deserialize(
+ cls,
+ file,
+ resolution=Resolution(file["width"], file["height"]) if file.get("hasVideo") else None,
+ )
+
+
+class PeerTubeCrawler(PeerTubeGenericCrawler):
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ("peertube",)
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.py
index 84c64700f..5200be938 100644
--- a/cyberdrop_dl/scrape_mapper.py
+++ b/cyberdrop_dl/scrape_mapper.py
@@ -4,6 +4,7 @@ import asyncio
import contextlib
import dataclasses
import logging
+import sys
import time
from pathlib import Path
from typing import TYPE_CHECKING, Literal, Self, override
@@ -29,7 +30,7 @@ from cyberdrop_dl.utils import remove_trailing_slash
from cyberdrop_dl.utils._url import matches_any_host
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Generator, Iterable, Iterator
+ from collections.abc import AsyncGenerator, Generator, Iterator
from cyberdrop_dl.clients.jd.client import JDownloader
from cyberdrop_dl.config import Config
@@ -43,10 +44,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-def _filter_by_domain(url: AbsoluteHttpURL, domains: Iterable[str]) -> bool:
- return any(domain in url.host for domain in domains)
-
-
@dataclasses.dataclass(slots=True, eq=False)
class CrawlerFactory:
manager: Manager
@@ -59,6 +56,12 @@ class CrawlerFactory:
return f"<{type(self).__name__}(instances={len(self._instances):,})>"
def __getitem__[CrawlerT: Crawler](self, obj: type[CrawlerT]) -> CrawlerT:
+ instance = self.get(obj)
+ if instance is None:
+ raise KeyError(obj)
+ return instance
+
+ def __call__[CrawlerT: Crawler](self, obj: type[CrawlerT]) -> CrawlerT:
instance = self.get(obj)
if instance is None:
instance = self._instances[obj] = obj(self.manager, self.task_mngr, self.tui)
@@ -149,7 +152,7 @@ class ScrapeMapper:
self.tui.scrape.get_queue = self._scrape_queue
self.tui.downloads.get_queue = self._download_queue
- def _init_crawlers(self) -> None:
+ async def _init_crawlers(self) -> None:
crawlers = get_crawlers_mapping()
self.crawlers.update(crawlers)
@@ -162,6 +165,33 @@ class ScrapeMapper:
logger.debug(msg)
_disable_crawlers_by_config(self.crawlers, *self.manager.config.crawlers.disabled)
+ await self._register_peertube()
+
+ async def _register_peertube(self) -> None:
+ # User may have disabled peertube
+ if "peertube" not in self.crawlers:
+ return
+
+ if "pytest" in sys.modules:
+ return
+
+ from cyberdrop_dl.crawlers._peertube import PeerTubeCrawler
+
+ for url in self.manager.config.crawlers.generic.peertube:
+ if other := _best_match(self.crawlers, url.host):
+ msg = GENERIC_MAP_ERROR.format(url, PeerTubeCrawler.NAME, other.NAME)
+ logger.error(msg)
+ continue
+
+ self.crawlers[url.host] = PeerTubeCrawler
+
+ peertube = self._factory(PeerTubeCrawler)
+ for host in await peertube.get_instances():
+ crawler = self.crawlers.setdefault(host, PeerTubeCrawler)
+ if crawler.DOMAIN == PeerTubeCrawler.DOMAIN:
+ continue
+
+ logger.warning("Found PeerTube site '%s' mapped to a non PeerTube crawler: %s", host, crawler.INFO)
@contextlib.asynccontextmanager
async def __call__(self) -> AsyncGenerator[Self]:
@@ -217,7 +247,7 @@ class ScrapeMapper:
async def __async_init__(self) -> None:
if self._ready:
return
- self._init_crawlers()
+ await self._init_crawlers()
try:
await self._jdownloader.connect(self.manager.http_client)
except Exception:
@@ -265,7 +295,7 @@ class ScrapeMapper:
async def _send_to_crawler(self, scrape_item: ScrapeItem) -> None:
if cls := _best_match(self.crawlers, scrape_item.url.host):
- crawler = self._factory[cls]
+ crawler = self._factory(cls)
await crawler.__async_init__()
if crawler.__url_config__.trim:
scrape_item.url = remove_trailing_slash(scrape_item.url)
@@ -330,6 +360,13 @@ def get_crawlers_mapping() -> dict[str, type[Crawler]]:
return crawlers_map
+GENERIC_MAP_ERROR = (
+ "Unable to assign {} to generic crawler {}. "
+ "URL conflicts with URL format of builtin crawler {}. "
+ "URL will be ignored"
+)
+
+
def register_crawler(
crawlers_map: dict[str, type[Crawler]],
crawler: type[Crawler],
@@ -343,11 +380,7 @@ def register_crawler(
if not other and (match := _best_match(crawlers_map, crawler.PRIMARY_URL.host)):
other = match
if other:
- msg = (
- f"Unable to assign {crawler.PRIMARY_URL} to generic crawler {crawler.NAME}. "
- f"URL conflicts with URL format of builtin crawler {other.NAME}. "
- "URL will be ignored"
- )
+ msg = GENERIC_MAP_ERROR.format(crawler.PRIMARY_URL, crawler.NAME, other.NAME)
if from_user == "raise":
raise ValueError(msg)
logger.error(msg)
diff --git a/tests/crawlers/test_cases/peertube.py b/tests/crawlers/test_cases/peertube.py
new file mode 100644
index 000000000..f230dcc4c
--- /dev/null
+++ b/tests/crawlers/test_cases/peertube.py
@@ -0,0 +1,52 @@
+DOMAIN = "peertube"
+TEST_CASES = [
+ {
+ "url": "https://framatube.org/w/217eefeb-883d-45be-b7fc-a788ad8507d3",
+ "results": [
+ {
+ "url": "https://framatube.org/lazy-static/video-captions/217eefeb-883d-45be-b7fc-a788ad8507d3-fr.vtt",
+ "filename": "What is PeerTube (english subtitles) [217eefeb-883d-45be-b7fc-a788ad8507d3][1080p@24fps].fr.vtt",
+ "debrid_url": None,
+ "original_filename": "French",
+ "referer": "https://framatube.org/w/217eefeb-883d-45be-b7fc-a788ad8507d3#What%20is%20PeerTube%20(english%20subtitles)%20%5B217eefeb-883d-45be-b7fc-a788ad8507d3%5D%5B1080p@24fps%5D.fr.vtt",
+ "album_id": None,
+ "uploaded_at": 1527168124,
+ "download_folder": "re:Loose Files (Framatube.org)",
+ "thumbnail": None,
+ },
+ {
+ "url": "https://framatube.org/lazy-static/video-captions/217eefeb-883d-45be-b7fc-a788ad8507d3-de.vtt",
+ "filename": "What is PeerTube (english subtitles) [217eefeb-883d-45be-b7fc-a788ad8507d3][1080p@24fps].de.vtt",
+ "debrid_url": None,
+ "original_filename": "German",
+ "referer": "https://framatube.org/w/217eefeb-883d-45be-b7fc-a788ad8507d3#What%20is%20PeerTube%20(english%20subtitles)%20%5B217eefeb-883d-45be-b7fc-a788ad8507d3%5D%5B1080p@24fps%5D.de.vtt",
+ "album_id": None,
+ "uploaded_at": 1527168124,
+ "download_folder": "re:Loose Files (Framatube.org)",
+ "thumbnail": None,
+ },
+ {
+ "url": "https://framatube.org/lazy-static/video-captions/217eefeb-883d-45be-b7fc-a788ad8507d3-en.vtt",
+ "filename": "What is PeerTube (english subtitles) [217eefeb-883d-45be-b7fc-a788ad8507d3][1080p@24fps].en.vtt",
+ "debrid_url": None,
+ "original_filename": "English",
+ "referer": "https://framatube.org/w/217eefeb-883d-45be-b7fc-a788ad8507d3#What%20is%20PeerTube%20(english%20subtitles)%20%5B217eefeb-883d-45be-b7fc-a788ad8507d3%5D%5B1080p@24fps%5D.en.vtt",
+ "album_id": None,
+ "uploaded_at": 1527168124,
+ "download_folder": "re:Loose Files (Framatube.org)",
+ "thumbnail": None,
+ },
+ {
+ "url": "https://framatube.org/static/streaming-playlists/hls/217eefeb-883d-45be-b7fc-a788ad8507d3/a109ecd1-f2f5-4da7-a17e-808d5e5e9ec8-1080-fragmented.mp4",
+ "filename": "What is PeerTube (english subtitles) [217eefeb-883d-45be-b7fc-a788ad8507d3][1080p@24fps].mp4",
+ "debrid_url": None,
+ "original_filename": "What is PeerTube? (english subtitles)",
+ "referer": "https://framatube.org/w/217eefeb-883d-45be-b7fc-a788ad8507d3",
+ "album_id": None,
+ "uploaded_at": 1527168124,
+ "download_folder": "re:Loose Files (Framatube.org)",
+ "thumbnail": "https://framatube.org/lazy-static/thumbnails/8f59e65b-89a1-4105-a5bd-4183b252efcf.jpg",
+ },
+ ],
+ },
+]
diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.py
index 1555112cf..a37267869 100644
--- a/tests/crawlers/test_crawlers.py
+++ b/tests/crawlers/test_crawlers.py
@@ -60,7 +60,7 @@ async def test_crawler(running_manager: Manager, test_case: test_cases.CrawlerTe
None,
)
assert cls, f"{test_case.domain} is not a valid crawler domain. Test case is invalid"
- crawler = scrape_mapper._factory[cls]
+ crawler = scrape_mapper._factory(cls)
await crawler.__async_init__()
item = ScrapeItem.from_url(crawler.parse_url(test_case.url))
item.download_folder = running_manager.config.download_folder