Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add generic video crawler
NTFSvolumea3e6825a1b87
diff --git a/cyberdrop_dl/config/crawlers.py b/cyberdrop_dl/config/crawlers.pyindex fbc8c64d9..618bc5bc9 100644--- a/cyberdrop_dl/config/crawlers.py+++ b/cyberdrop_dl/config/crawlers.py@@ -89,6 +89,7 @@ class GenericCrawlers(ConfigModel):discourse: tuple[HttpURL, ...] = ()chevereto: tuple[HttpURL, ...] = ()kvs: tuple[HttpURL, ...] = ()+ video: tuple[HttpURL, ...] = ()class Crawlers(ConfigGroup, name=None):diff --git a/cyberdrop_dl/crawlers/_video.py b/cyberdrop_dl/crawlers/_video.pynew file mode 100644index 000000000..df127e770--- /dev/null+++ b/cyberdrop_dl/crawlers/_video.py@@ -0,0 +1,36 @@+from __future__ import annotations++from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl.crawlers.crawler import Crawler, URLConfig+from cyberdrop_dl.utils import css+from cyberdrop_dl.utils.errors import error_handling_wrapper++if TYPE_CHECKING:+ from cyberdrop_dl.url_objects import ScrapeItem++[email protected]_path_builder("url")+@URLConfig(trim=False)+class GenericVideoCrawler(Crawler, is_generic=True):+ SUPPORTED_PATHS: ClassVar[dict[str, str | tuple[str, ...]]] = {"Video": "/..."}++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ await self.video(scrape_item)++ @error_handling_wrapper+ async def video(self, scrape_item: ScrapeItem) -> None:+ if await self.check_complete_from_referer(scrape_item.url):+ return++ soup = await self.request_soup(scrape_item.url)+ src = self.parse_url(css.select(soup, "video source", "src"))++ _, ext = self.get_filename_and_ext(src.name, assume_ext=".mp4")+ scrape_item.referer = scrape_item.url++ if ext == ".m3u8":+ await self.generic_m3u8(scrape_item, src)+ return++ await self.direct_file(scrape_item, src)diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 349a1c66c..2ff87ae9e 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -936,6 +936,35 @@ class Crawler(HTTPMixin, HLSMixin, ABC):))+ async def generic_m3u8(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:+ url = url or scrape_item.url+ referer = scrape_item.get_referer()+ with self.catch_errors(url):+ if referer and await self.check_complete_from_referer(referer):+ return++ if await self.check_complete(url):+ return++ headers = {"Referer": str(referer)} if referer else {}+ m3u8, info = await self.request_m3u8(url, headers=headers)+ name = url.name or url.parent.name+ filename = self.create_custom_filename(+ url.path,+ ext := ".mp4",+ resolution=info and info.resolution,+ video_codec=info and info.codecs.video,+ audio_codec=info and info.codecs.audio,+ )+ await self.handle_file(+ url,+ scrape_item,+ name,+ ext,+ m3u8=m3u8,+ custom_filename=filename,+ )+@HTTPConfig(rate_limit=(25, 1))class API(HTTPMixin, ABC):diff --git a/cyberdrop_dl/crawlers/http_direct.py b/cyberdrop_dl/crawlers/http_direct.pyindex 559c2f529..638c22a63 100644--- a/cyberdrop_dl/crawlers/http_direct.py+++ b/cyberdrop_dl/crawlers/http_direct.py@@ -7,7 +7,6 @@ from cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.downloader.http import Downloaderfrom cyberdrop_dl.exceptions import NoExtensionErrorfrom cyberdrop_dl.filepath import get_filename_and_ext-from cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem@@ -28,7 +27,9 @@ class DirectHttpFileCrawler(Crawler, is_generic=True):filename, ext = get_filename_and_ext(scrape_item.url.name, xenforo=True)if ext == ".m3u8":- return await self.m3u8(scrape_item)+ scrape_item.append_folders("Loose Files")+ scrape_item.part_of_album = True+ return await self.generic_m3u8(scrape_item)if ext not in FileExt.MEDIA:raise ValueError@@ -42,30 +43,3 @@ class DirectHttpFileCrawler(Crawler, is_generic=True):ext,custom_filename=filename,)-- @error_handling_wrapper- async def m3u8(self, scrape_item: ScrapeItem) -> None:- if await self.check_complete_from_referer(scrape_item.url):- return-- scrape_item.append_folders("Loose Files")- scrape_item.part_of_album = True- url = scrape_item.url- headers = {"Referer": str(referer)} if (referer := scrape_item.get_referer()) else {}- m3u8, info = await self.request_m3u8(url, headers=headers)- name = url.name or url.parent.name- filename = self.create_custom_filename(- scrape_item.url.path,- ext := ".mp4",- resolution=info and info.resolution,- video_codec=info and info.codecs.video,- audio_codec=info and info.codecs.audio,- )- await self.handle_file(- scrape_item.url,- scrape_item,- name,- ext,- m3u8=m3u8,- custom_filename=filename,- )diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex b3fb7805d..ca0bcbf4d 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -355,6 +355,11 @@ def _create_generic_crawlers(generics_config: GenericCrawlers) -> Generator[typeyield from create_crawlers(generics_config.kvs, GenericKVSCrawler)+ if generics_config.video:+ from cyberdrop_dl.crawlers._video import GenericVideoCrawler++ yield from create_crawlers(generics_config.video, GenericVideoCrawler)+def _disable_crawlers_by_config(current_crawlers: dict[str, type[Crawler]], *crawlers_to_disable: str) -> None:if not crawlers_to_disable: