Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: handle unavailable videos (SendVid)
NTFSvolumecb9fff052caa
diff --git a/cyberdrop_dl/clients/etag.py b/cyberdrop_dl/clients/etag.pyindex 9c659aa58..d9b344bcb 100644--- a/cyberdrop_dl/clients/etag.py+++ b/cyberdrop_dl/clients/etag.py@@ -33,6 +33,10 @@ for domain, params in {("19fdf2cd6-383c-5a4cd5b6710ed", "Image not found"),("383c-5a4cd5b6710ed", "Image not found"),],+ "SendVid": [+ ("6ab3d9ed-8fb5", "temporarily unavailable", HTTPStatus.SERVICE_UNAVAILABLE),+ ("6ab3db3e-3d0a", "thumbnail temporarily unavailable", HTTPStatus.SERVICE_UNAVAILABLE),+ ],}.items():for param in params:new_etag = ETag(domain, *param)diff --git a/cyberdrop_dl/crawlers/imagepond.py b/cyberdrop_dl/crawlers/imagepond.pyindex faabeb5a4..13c6cd12e 100644--- a/cyberdrop_dl/crawlers/imagepond.py+++ b/cyberdrop_dl/crawlers/imagepond.py@@ -125,7 +125,7 @@ class ImagePondCrawler(Crawler):if soup.select_one(Selector.EXPIRED):redirect = await self.request_redirect(resp.url / "thumb/300")if "thumb" in redirect.parts or redirect.name.startswith("thumb_") or not self.is_subdomain(redirect):- raise ScrapeError(404, "Content Expired")+ raise ScrapeError(410, "Content Expired")await self.direct_file(scrape_item, redirect)returndiff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.pyindex d2b3f4b23..cc835a36e 100644--- a/cyberdrop_dl/crawlers/sendvid.py+++ b/cyberdrop_dl/crawlers/sendvid.py@@ -1,6 +1,6 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, ClassVar, finalfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeError@@ -12,67 +12,63 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem+@finalclass Selector:+ VIDEO = "#video_container"VIDEO_SRC = "video > source#video_source"TITLE = "p.video-title"-REQUIRED_QUERY_PARAMS = "validfrom", "validto", "rate", "ip", "hash"--class SendVidCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Videos": "/...",- "Embeds": "/embed/...",- "Direct links": "",+ "Videos": "/<video_id>",+ "Embeds": "/embed/<video_id>",+ "Direct links": "https://videos<x>.sendvid.com/...",}- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://sendvid.com/")+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://sendvid.com")DOMAIN: ClassVar[str] = "sendvid"FOLDER_DOMAIN: ClassVar[str] = "SendVid"async def fetch(self, scrape_item: ScrapeItem) -> None:- scrape_item.url = _get_streaming_url(scrape_item.url)- await self.video(scrape_item)+ match scrape_item.url.parts[1:]:+ case [video_id]:+ await self.video(scrape_item, video_id)+ case _:+ raise ValueError++ @classmethod+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ url = super().transform_url(url)+ match url.parts[1:]:+ case ["embed", video_id]:+ return url.origin() / video_id+ case [_, *_] if cls.is_subdomain(url):+ video_id = url.name.split(".", 1)[0]+ return cls.PRIMARY_URL / video_id+ case _:+ return url@error_handling_wrapper- async def video(self, scrape_item: ScrapeItem) -> None:+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:if await self.check_complete_from_referer(scrape_item.url):returnsoup = await self.request_soup(scrape_item.url)-+ video = css.select(soup, Selector.VIDEO)title = css.select_text(soup, Selector.TITLE)- try:- link_str: str = css.select(soup, Selector.VIDEO_SRC, "src")- except AssertionError:- raise ScrapeError(422, "Couldn't find video source") from None- link = self.parse_url(link_str)- await self._video(scrape_item, link, title)+ src = self.parse_url(css.select(video, Selector.VIDEO_SRC, "src"))- async def _video(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None, title: str = "") -> None:- link = link or scrape_item.url- canonical_url = link.with_query(None)-- if not all(param in link.query for param in REQUIRED_QUERY_PARAMS):- msg = f"URL is missing some of the required parameters: {REQUIRED_QUERY_PARAMS}"+ if not all(map(src.query.get, required := ("validfrom", "validto", "rate", "hash"))):+ msg = f"URL {src} is missing some of the required parameters: {required}"raise ScrapeError(401, msg)- filename, ext = self.get_filename_and_ext(link.name)- custom_filename = self.get_filename_and_ext(f"{title}{ext}")[0] if title else None+ filename, ext = self.get_filename_and_ext(src.name)await self.handle_file(- canonical_url, scrape_item, filename, ext, debrid_link=link, custom_filename=custom_filename+ src.with_query(None),+ scrape_item,+ src.name,+ ext,+ debrid_link=src,+ custom_filename=self.create_custom_filename(title or filename, ext, file_id=video_id),+ thumbnail=css.attr(video, "poster"),)---def _get_streaming_url(url: AbsoluteHttpURL) -> AbsoluteHttpURL:- if is_cdn(url):- video_id = url.name.split(".", 1)[0]- return SendVidCrawler.PRIMARY_URL.with_path(video_id)-- if "embed" in url.parts:- return url.with_path(url.path.replace("/embed/", "/"), keep_query=True, keep_fragment=True)- return url---def is_cdn(url: AbsoluteHttpURL) -> bool:- return all(p in url.host for p in (SendVidCrawler.PRIMARY_URL.host, "."))