Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add support for generic M3U8 URLs

NTFSvolume2bd9fffe944e
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.py
index fc61c1582..6ea630d17 100644
--- a/cyberdrop_dl/clients/http.py
+++ b/cyberdrop_dl/clients/http.py
@@ -291,8 +291,9 @@ class HTTPClient:
@contextlib.asynccontextmanager
async def rate_limit_ctx(self, domain: str, json_check: JSONCheck | None = None) -> AsyncGenerator[None]:
+ limiter = self.limiter.per_domain.get(domain, contextlib.nullcontext())
with enter_context(JSON_CHECK, json_check):
- async with self.limiter.per_domain[domain], self.limiter.global_:
+ async with limiter, self.limiter.global_:
yield
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 530b0a007..1ae86c04d 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -209,7 +209,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
def apply[T: Crawler](cls: type[T]) -> type[T]:
cls.__db_path__ = staticmethod(_DB_PATH_BUILDERS[key])
- return URLConfig(ignore_fragment="frag" not in key)(cls)
+ return URLConfig(ignore_fragment=not ("frag" in key or key == "url"))(cls)
return apply
@@ -507,6 +507,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
referer: AbsoluteHttpURL | None = None,
frag: str | None = None,
thumbnail: AbsoluteHttpURL | None = None,
+ headers: Mapping[str, str] | None = None,
) -> None:
"""Creates a MediaItem and hands it off to the downloader.
@@ -533,6 +534,9 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
)
media_item.headers.update(self._prepare_headers(scrape_item))
+ if headers:
+ media_item.headers.update(headers)
+
if metadata:
media_item.metadata = metadata
diff --git a/cyberdrop_dl/crawlers/http_direct.py b/cyberdrop_dl/crawlers/http_direct.py
index 51815a3fc..cb9b4610d 100644
--- a/cyberdrop_dl/crawlers/http_direct.py
+++ b/cyberdrop_dl/crawlers/http_direct.py
@@ -7,11 +7,13 @@ from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.downloader.http import Downloader
from cyberdrop_dl.exceptions import NoExtensionError
from cyberdrop_dl.filepath import get_filename_and_ext
+from cyberdrop_dl.utils.errors import error_handling_wrapper
if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
[email protected]_path_builder("url")
class DirectHttpFileCrawler(Crawler, is_generic=True):
DOMAIN: ClassVar[str] = "no_crawler"
@@ -25,6 +27,9 @@ class DirectHttpFileCrawler(Crawler, is_generic=True):
except NoExtensionError:
filename, ext = get_filename_and_ext(scrape_item.url.name, xenforo=True)
+ if ext == ".m3u8":
+ return await self.m3u8(scrape_item)
+
if ext not in FileExt.MEDIA:
raise ValueError
@@ -37,3 +42,30 @@ class DirectHttpFileCrawler(Crawler, is_generic=True):
ext,
custom_filename=filename,
)
+
+ @error_handling_wrapper
+ async def m3u8(self, scrape_item: ScrapeItem) -> None:
+ if await self.check_complete_from_referer(scrape_item.url):
+ return
+
+ scrape_item.append_folders("Loose Files")
+ scrape_item.part_of_album = True
+ url = scrape_item.url
+ headers = {"Referer": str(scrape_item.referer)} if scrape_item.referer else {}
+ m3u8, info = await self.request_m3u8(url, headers=headers)
+ name = url.name or url.parent.name
+ filename = self.create_custom_filename(
+ scrape_item.url.path,
+ ext := ".mp4",
+ resolution=info and info.resolution,
+ video_codec=info and info.codecs.video,
+ audio_codec=info and info.codecs.audio,
+ )
+ await self.handle_file(
+ scrape_item.url,
+ scrape_item,
+ name,
+ ext,
+ m3u8=m3u8,
+ custom_filename=filename,
+ )