Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add support for generic M3U8 URLs
NTFSvolume2bd9fffe944e
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex fc61c1582..6ea630d17 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -291,8 +291,9 @@ class HTTPClient:@contextlib.asynccontextmanagerasync def rate_limit_ctx(self, domain: str, json_check: JSONCheck | None = None) -> AsyncGenerator[None]:+ limiter = self.limiter.per_domain.get(domain, contextlib.nullcontext())with enter_context(JSON_CHECK, json_check):- async with self.limiter.per_domain[domain], self.limiter.global_:+ async with limiter, self.limiter.global_:yielddiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 530b0a007..1ae86c04d 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -209,7 +209,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):def apply[T: Crawler](cls: type[T]) -> type[T]:cls.__db_path__ = staticmethod(_DB_PATH_BUILDERS[key])- return URLConfig(ignore_fragment="frag" not in key)(cls)+ return URLConfig(ignore_fragment=not ("frag" in key or key == "url"))(cls)return apply@@ -507,6 +507,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):referer: AbsoluteHttpURL | None = None,frag: str | None = None,thumbnail: AbsoluteHttpURL | None = None,+ headers: Mapping[str, str] | None = None,) -> None:"""Creates a MediaItem and hands it off to the downloader.@@ -533,6 +534,9 @@ class Crawler(HTTPMixin, HLSMixin, ABC):)media_item.headers.update(self._prepare_headers(scrape_item))+ if headers:+ media_item.headers.update(headers)+if metadata:media_item.metadata = metadatadiff --git a/cyberdrop_dl/crawlers/http_direct.py b/cyberdrop_dl/crawlers/http_direct.pyindex 51815a3fc..cb9b4610d 100644--- a/cyberdrop_dl/crawlers/http_direct.py+++ b/cyberdrop_dl/crawlers/http_direct.py@@ -7,11 +7,13 @@ from cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.downloader.http import Downloaderfrom cyberdrop_dl.exceptions import NoExtensionErrorfrom cyberdrop_dl.filepath import get_filename_and_ext+from cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem[email protected]_path_builder("url")class DirectHttpFileCrawler(Crawler, is_generic=True):DOMAIN: ClassVar[str] = "no_crawler"@@ -25,6 +27,9 @@ class DirectHttpFileCrawler(Crawler, is_generic=True):except NoExtensionError:filename, ext = get_filename_and_ext(scrape_item.url.name, xenforo=True)+ if ext == ".m3u8":+ return await self.m3u8(scrape_item)+if ext not in FileExt.MEDIA:raise ValueError@@ -37,3 +42,30 @@ class DirectHttpFileCrawler(Crawler, is_generic=True):ext,custom_filename=filename,)++ @error_handling_wrapper+ async def m3u8(self, scrape_item: ScrapeItem) -> None:+ if await self.check_complete_from_referer(scrape_item.url):+ return++ scrape_item.append_folders("Loose Files")+ scrape_item.part_of_album = True+ url = scrape_item.url+ headers = {"Referer": str(scrape_item.referer)} if scrape_item.referer else {}+ m3u8, info = await self.request_m3u8(url, headers=headers)+ name = url.name or url.parent.name+ filename = self.create_custom_filename(+ scrape_item.url.path,+ ext := ".mp4",+ resolution=info and info.resolution,+ video_codec=info and info.codecs.video,+ audio_codec=info and info.codecs.audio,+ )+ await self.handle_file(+ scrape_item.url,+ scrape_item,+ name,+ ext,+ m3u8=m3u8,+ custom_filename=filename,+ )