Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add odysee support
NTFSvolumeec3bfee55319
diff --git a/cyberdrop_dl/crawlers/odysee.py b/cyberdrop_dl/crawlers/odysee.pynew file mode 100644index 000000000..87f7c20ef--- /dev/null+++ b/cyberdrop_dl/crawlers/odysee.py@@ -0,0 +1,138 @@+from __future__ import annotations++import dataclasses+import time+from typing import TYPE_CHECKING, Any, ClassVar, Self, override++from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths+from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError+from cyberdrop_dl.mediaprops import Resolution+from cyberdrop_dl.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import parse_url+from cyberdrop_dl.utils.dataclass import deserialize+from cyberdrop_dl.utils.errors import error_handling_wrapper++if TYPE_CHECKING:+ from cyberdrop_dl.url_objects import ScrapeItem++[email protected]_path_builder("url")+class OdyseeCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Video": ("@channel:uri",),+ }++ DOMAIN: ClassVar[str] = "odysee"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://odysee.com")++ def __post_init__(self) -> None:+ self.api: LBRYAPI = LBRYAPI.from_crawler(self)++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case [channel, _] if channel.startswith("@"):+ uri = "lbry:/" + scrape_item.url.path.replace(":", "#")+ await self.video(scrape_item, uri)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def video(self, scrape_item: ScrapeItem, uri: str) -> None:+ if await self.check_complete_from_referer(scrape_item.url):+ return++ video = await self.api.resolve(uri)+ src = await self.api.stream(uri)+ filename = self.create_custom_filename(+ video.title,+ ext := ".mp4",+ file_id=video.id,+ resolution=Resolution(video.width, video.height),+ )+ scrape_item.uploaded_at = video.release_time+ await self.handle_file(+ AbsoluteHttpURL(uri),+ scrape_item,+ video.title,+ ext,+ custom_filename=filename,+ debrid_link=src,+ metadata=video,+ thumbnail=video.thumbnail,+ )++ @override+ @classmethod+ def get_filename_and_ext( # pyright: ignore[reportIncompatibleMethodOverride]+ cls,+ filename: str,+ *,+ assume_ext: str | None = ".mp4",+ mime_type: str | None = None,+ ) -> tuple[str, str]:+ try:+ return super().get_filename_and_ext(filename, mime_type=mime_type, assume_ext=assume_ext)+ except NoExtensionError:+ return super().get_filename_and_ext(filename + ".webp", mime_type=mime_type, assume_ext=assume_ext)++[email protected](slots=True, frozen=True)+class Video:+ id: str+ title: str+ height: int+ width: int+ release_time: int+ thumbnail: AbsoluteHttpURL++ @classmethod+ def parse(cls, stream: dict[str, Any]) -> Self:+ video = stream["value"] | stream["value"]["video"]+ return deserialize(+ cls,+ video,+ id=stream["claim_id"],+ thumbnail=parse_url(video["thumbnail"]["url"]),+ release_time=int(video["release_time"]),+ )+++class LBRYAPI(API):+ PROXY: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api.na-backend.odysee.com/api/v1/proxy")++ async def request_json_rpc(self, method: str, **params: Any) -> dict[str, Any]:+ resp = await self.request_json(+ self.PROXY,+ headers={"Content-Type": "application/json-rpc"},+ json={+ "id": int(time.time()),+ "jsonrpc": "2.0",+ "method": method,+ "params": params,+ },+ )++ if error := resp.get("error"):+ msg = f"{error.get('code')} - {error.get('message')}"+ raise ScrapeError(422, msg)++ return resp["result"]++ async def resolve(self, url: str) -> Video:+ resp = await self.request_json_rpc("resolve", urls=url)+ stream = resp[url]+ if error := stream.get("error"):+ code = 404 if error.get("name") == "NOT_FOUND" else 422+ raise ScrapeError(code, error.get("text") or str(error))++ stream_type = stream["value"].get("stream_type", "livestream")+ if stream_type != "video":+ raise ScrapeError(422, f"{stream_type = } is not supported")+ return Video.parse(stream)++ async def stream(self, url: str) -> AbsoluteHttpURL:+ resp = await self.request_json_rpc("get", enviroment=None, uri=url)+ stream_url = parse_url(resp["streaming_url"])+ if stream_url.suffix == ".m3u8":+ raise ValueError("Unsupport m3u8 stream")+ return stream_urldiff --git a/tests/crawlers/test_cases/odysee.py b/tests/crawlers/test_cases/odysee.pynew file mode 100644index 000000000..1159a567c--- /dev/null+++ b/tests/crawlers/test_cases/odysee.py@@ -0,0 +1,28 @@+DOMAIN = "odysee"+TEST_CASES = [+ {+ "url": "https://odysee.com/@fireship:6/robot-demos-have-a-dirty-little:c",+ "results": [+ {+ "url": "lbry://fireship/#6/robot-demos-have-a-dirty-little%23c",+ "filename": "Robot demos have a dirty little secret... [c6fb0fe6ef63bdc02c21d9f457ee045928f79905][1080p].mp4",+ "debrid_url": "https://player.odycdn.com/v6/streams/c6fb0fe6ef63bdc02c21d9f457ee045928f79905/a19c98.mp4",+ "original_filename": "Robot demos have a dirty little secret...",+ "referer": "https://odysee.com/@fireship:6/robot-demos-have-a-dirty-little:c",+ "album_id": None,+ "uploaded_at": 1786464608,+ "download_folder": "re:Loose Files (Odysee)",+ },+ {+ "url": "https://thumbnails.lbry.com/aB5LGrHISqY",+ "filename": "Robot demos have a dirty little secret... [c6fb0fe6ef63bdc02c21d9f457ee045928f79905][1080p.webp",+ "debrid_url": None,+ "original_filename": "Robot demos have a dirty little secret... [c6fb0fe6ef63bdc02c21d9f457ee045928f79905][1080p]_thumb.webp",+ "referer": "https://odysee.com/@fireship:6/robot-demos-have-a-dirty-little:c#thumbnail",+ "album_id": None,+ "uploaded_at": 1786464608,+ "download_folder": "re:Loose Files (Odysee)",+ },+ ],+ },+]