Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add generic video crawler

NTFSvolumea3e6825a1b87
diff --git a/cyberdrop_dl/config/crawlers.py b/cyberdrop_dl/config/crawlers.py
index fbc8c64d9..618bc5bc9 100644
--- a/cyberdrop_dl/config/crawlers.py
+++ b/cyberdrop_dl/config/crawlers.py
@@ -89,6 +89,7 @@ class GenericCrawlers(ConfigModel):
discourse: tuple[HttpURL, ...] = ()
chevereto: tuple[HttpURL, ...] = ()
kvs: tuple[HttpURL, ...] = ()
+ video: tuple[HttpURL, ...] = ()
class Crawlers(ConfigGroup, name=None):
diff --git a/cyberdrop_dl/crawlers/_video.py b/cyberdrop_dl/crawlers/_video.py
new file mode 100644
index 000000000..df127e770
--- /dev/null
+++ b/cyberdrop_dl/crawlers/_video.py
@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl.crawlers.crawler import Crawler, URLConfig
+from cyberdrop_dl.utils import css
+from cyberdrop_dl.utils.errors import error_handling_wrapper
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.url_objects import ScrapeItem
+
+
[email protected]_path_builder("url")
+@URLConfig(trim=False)
+class GenericVideoCrawler(Crawler, is_generic=True):
+ SUPPORTED_PATHS: ClassVar[dict[str, str | tuple[str, ...]]] = {"Video": "/..."}
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ await self.video(scrape_item)
+
+ @error_handling_wrapper
+ async def video(self, scrape_item: ScrapeItem) -> None:
+ if await self.check_complete_from_referer(scrape_item.url):
+ return
+
+ soup = await self.request_soup(scrape_item.url)
+ src = self.parse_url(css.select(soup, "video source", "src"))
+
+ _, ext = self.get_filename_and_ext(src.name, assume_ext=".mp4")
+ scrape_item.referer = scrape_item.url
+
+ if ext == ".m3u8":
+ await self.generic_m3u8(scrape_item, src)
+ return
+
+ await self.direct_file(scrape_item, src)
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 349a1c66c..2ff87ae9e 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -936,6 +936,35 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
)
)
+ async def generic_m3u8(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:
+ url = url or scrape_item.url
+ referer = scrape_item.get_referer()
+ with self.catch_errors(url):
+ if referer and await self.check_complete_from_referer(referer):
+ return
+
+ if await self.check_complete(url):
+ return
+
+ headers = {"Referer": str(referer)} if referer else {}
+ m3u8, info = await self.request_m3u8(url, headers=headers)
+ name = url.name or url.parent.name
+ filename = self.create_custom_filename(
+ url.path,
+ ext := ".mp4",
+ resolution=info and info.resolution,
+ video_codec=info and info.codecs.video,
+ audio_codec=info and info.codecs.audio,
+ )
+ await self.handle_file(
+ url,
+ scrape_item,
+ name,
+ ext,
+ m3u8=m3u8,
+ custom_filename=filename,
+ )
+
@HTTPConfig(rate_limit=(25, 1))
class API(HTTPMixin, ABC):
diff --git a/cyberdrop_dl/crawlers/http_direct.py b/cyberdrop_dl/crawlers/http_direct.py
index 559c2f529..638c22a63 100644
--- a/cyberdrop_dl/crawlers/http_direct.py
+++ b/cyberdrop_dl/crawlers/http_direct.py
@@ -7,7 +7,6 @@ from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.downloader.http import Downloader
from cyberdrop_dl.exceptions import NoExtensionError
from cyberdrop_dl.filepath import get_filename_and_ext
-from cyberdrop_dl.utils.errors import error_handling_wrapper
if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
@@ -28,7 +27,9 @@ class DirectHttpFileCrawler(Crawler, is_generic=True):
filename, ext = get_filename_and_ext(scrape_item.url.name, xenforo=True)
if ext == ".m3u8":
- return await self.m3u8(scrape_item)
+ scrape_item.append_folders("Loose Files")
+ scrape_item.part_of_album = True
+ return await self.generic_m3u8(scrape_item)
if ext not in FileExt.MEDIA:
raise ValueError
@@ -42,30 +43,3 @@ class DirectHttpFileCrawler(Crawler, is_generic=True):
ext,
custom_filename=filename,
)
-
- @error_handling_wrapper
- async def m3u8(self, scrape_item: ScrapeItem) -> None:
- if await self.check_complete_from_referer(scrape_item.url):
- return
-
- scrape_item.append_folders("Loose Files")
- scrape_item.part_of_album = True
- url = scrape_item.url
- headers = {"Referer": str(referer)} if (referer := scrape_item.get_referer()) else {}
- m3u8, info = await self.request_m3u8(url, headers=headers)
- name = url.name or url.parent.name
- filename = self.create_custom_filename(
- scrape_item.url.path,
- ext := ".mp4",
- resolution=info and info.resolution,
- video_codec=info and info.codecs.video,
- audio_codec=info and info.codecs.audio,
- )
- await self.handle_file(
- scrape_item.url,
- scrape_item,
- name,
- ext,
- m3u8=m3u8,
- custom_filename=filename,
- )
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.py
index b3fb7805d..ca0bcbf4d 100644
--- a/cyberdrop_dl/scrape_mapper.py
+++ b/cyberdrop_dl/scrape_mapper.py
@@ -355,6 +355,11 @@ def _create_generic_crawlers(generics_config: GenericCrawlers) -> Generator[type
yield from create_crawlers(generics_config.kvs, GenericKVSCrawler)
+ if generics_config.video:
+ from cyberdrop_dl.crawlers._video import GenericVideoCrawler
+
+ yield from create_crawlers(generics_config.video, GenericVideoCrawler)
+
def _disable_crawlers_by_config(current_crawlers: dict[str, type[Crawler]], *crawlers_to_disable: str) -> None:
if not crawlers_to_disable: