Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: handle unavailable videos (SendVid)

NTFSvolumecb9fff052caa
diff --git a/cyberdrop_dl/clients/etag.py b/cyberdrop_dl/clients/etag.py
index 9c659aa58..d9b344bcb 100644
--- a/cyberdrop_dl/clients/etag.py
+++ b/cyberdrop_dl/clients/etag.py
@@ -33,6 +33,10 @@ for domain, params in {
("19fdf2cd6-383c-5a4cd5b6710ed", "Image not found"),
("383c-5a4cd5b6710ed", "Image not found"),
],
+ "SendVid": [
+ ("6ab3d9ed-8fb5", "temporarily unavailable", HTTPStatus.SERVICE_UNAVAILABLE),
+ ("6ab3db3e-3d0a", "thumbnail temporarily unavailable", HTTPStatus.SERVICE_UNAVAILABLE),
+ ],
}.items():
for param in params:
new_etag = ETag(domain, *param)
diff --git a/cyberdrop_dl/crawlers/imagepond.py b/cyberdrop_dl/crawlers/imagepond.py
index faabeb5a4..13c6cd12e 100644
--- a/cyberdrop_dl/crawlers/imagepond.py
+++ b/cyberdrop_dl/crawlers/imagepond.py
@@ -125,7 +125,7 @@ class ImagePondCrawler(Crawler):
if soup.select_one(Selector.EXPIRED):
redirect = await self.request_redirect(resp.url / "thumb/300")
if "thumb" in redirect.parts or redirect.name.startswith("thumb_") or not self.is_subdomain(redirect):
- raise ScrapeError(404, "Content Expired")
+ raise ScrapeError(410, "Content Expired")
await self.direct_file(scrape_item, redirect)
return
diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.py
index d2b3f4b23..cc835a36e 100644
--- a/cyberdrop_dl/crawlers/sendvid.py
+++ b/cyberdrop_dl/crawlers/sendvid.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, final
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
@@ -12,67 +12,63 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
+@final
class Selector:
+ VIDEO = "#video_container"
VIDEO_SRC = "video > source#video_source"
TITLE = "p.video-title"
-REQUIRED_QUERY_PARAMS = "validfrom", "validto", "rate", "ip", "hash"
-
-
class SendVidCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Videos": "/...",
- "Embeds": "/embed/...",
- "Direct links": "",
+ "Videos": "/<video_id>",
+ "Embeds": "/embed/<video_id>",
+ "Direct links": "https://videos<x>.sendvid.com/...",
}
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://sendvid.com/")
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://sendvid.com")
DOMAIN: ClassVar[str] = "sendvid"
FOLDER_DOMAIN: ClassVar[str] = "SendVid"
async def fetch(self, scrape_item: ScrapeItem) -> None:
- scrape_item.url = _get_streaming_url(scrape_item.url)
- await self.video(scrape_item)
+ match scrape_item.url.parts[1:]:
+ case [video_id]:
+ await self.video(scrape_item, video_id)
+ case _:
+ raise ValueError
+
+ @classmethod
+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ url = super().transform_url(url)
+ match url.parts[1:]:
+ case ["embed", video_id]:
+ return url.origin() / video_id
+ case [_, *_] if cls.is_subdomain(url):
+ video_id = url.name.split(".", 1)[0]
+ return cls.PRIMARY_URL / video_id
+ case _:
+ return url
@error_handling_wrapper
- async def video(self, scrape_item: ScrapeItem) -> None:
+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
if await self.check_complete_from_referer(scrape_item.url):
return
soup = await self.request_soup(scrape_item.url)
-
+ video = css.select(soup, Selector.VIDEO)
title = css.select_text(soup, Selector.TITLE)
- try:
- link_str: str = css.select(soup, Selector.VIDEO_SRC, "src")
- except AssertionError:
- raise ScrapeError(422, "Couldn't find video source") from None
- link = self.parse_url(link_str)
- await self._video(scrape_item, link, title)
+ src = self.parse_url(css.select(video, Selector.VIDEO_SRC, "src"))
- async def _video(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None, title: str = "") -> None:
- link = link or scrape_item.url
- canonical_url = link.with_query(None)
-
- if not all(param in link.query for param in REQUIRED_QUERY_PARAMS):
- msg = f"URL is missing some of the required parameters: {REQUIRED_QUERY_PARAMS}"
+ if not all(map(src.query.get, required := ("validfrom", "validto", "rate", "hash"))):
+ msg = f"URL {src} is missing some of the required parameters: {required}"
raise ScrapeError(401, msg)
- filename, ext = self.get_filename_and_ext(link.name)
- custom_filename = self.get_filename_and_ext(f"{title}{ext}")[0] if title else None
+ filename, ext = self.get_filename_and_ext(src.name)
await self.handle_file(
- canonical_url, scrape_item, filename, ext, debrid_link=link, custom_filename=custom_filename
+ src.with_query(None),
+ scrape_item,
+ src.name,
+ ext,
+ debrid_link=src,
+ custom_filename=self.create_custom_filename(title or filename, ext, file_id=video_id),
+ thumbnail=css.attr(video, "poster"),
)
-
-
-def _get_streaming_url(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- if is_cdn(url):
- video_id = url.name.split(".", 1)[0]
- return SendVidCrawler.PRIMARY_URL.with_path(video_id)
-
- if "embed" in url.parts:
- return url.with_path(url.path.replace("/embed/", "/"), keep_query=True, keep_fragment=True)
- return url
-
-
-def is_cdn(url: AbsoluteHttpURL) -> bool:
- return all(p in url.host for p in (SendVidCrawler.PRIMARY_URL.host, "."))