Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add lulustream support
NTFSvolumed80fb1dbfd29
diff --git a/cyberdrop_dl/crawlers/lulustream.py b/cyberdrop_dl/crawlers/lulustream.pynew file mode 100644index 000000000..3f67519e2--- /dev/null+++ b/cyberdrop_dl/crawlers/lulustream.py@@ -0,0 +1,87 @@+from __future__ import annotations++import dataclasses+from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl import aio+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedDomains, SupportedPaths+from cyberdrop_dl.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import TextExtractor, css, js_unpacker+from cyberdrop_dl.utils.errors import error_handling_wrapper++if TYPE_CHECKING:+ from cyberdrop_dl.url_objects import ScrapeItem+++class LuluStreamCrawler(Crawler):+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (+ "lulustream",+ "luluvdo.com",+ "lulu.st",+ "luluvid.com",+ "cdn1.site",+ "streamhihi.com",+ "luluvdoo.com",+ )+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Video": (+ "/e/<video_id>",+ "/d/<video_id>",+ "/<video_id>",+ ),+ }+ DOMAIN: ClassVar[str] = "lulustream"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://lulustream.com")++ def __post_init__(self) -> None:+ self.api: LuluStreamAPI = LuluStreamAPI.from_crawler(self)++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["e" | "d", video_id] | [video_id]:+ await self.video(scrape_item, video_id)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:+ embed_url = self.PRIMARY_URL / "e" / video_id+ if await self.check_complete(embed_url):+ return++ video = await self.api.video(video_id)+ m3u8, info = await self.request_m3u8_playlist(video.m3u8)+ await self.handle_file(+ embed_url,+ scrape_item,+ video.title,+ ext := ".mp4",+ m3u8=m3u8,+ custom_filename=self.create_custom_filename(video.title, ext, resolution=info.resolution),+ thumbnail=video.thumb,+ )++[email protected](slots=True, frozen=True)+class Video:+ id: str+ title: str+ m3u8: AbsoluteHttpURL+ thumb: AbsoluteHttpURL+++class LuluStreamAPI(API):+ async def video(self, video_id: str) -> Video:+ embed, web = await aio.gather(+ self.request_soup(self.origin / "e" / video_id),+ self.request_soup(self.origin / "d" / video_id),+ fail_fast=False,+ )+ player = js_unpacker.unpack(js_unpacker.find(embed))+ extr = TextExtractor(player)+ return Video(+ id=video_id,+ title=css.select_text(web, "h1"),+ m3u8=self.parse_url(extr('{sources:[{file:"', '"')),+ thumb=self.parse_url(extr('image:"', '"')),+ )diff --git a/cyberdrop_dl/crawlers/streamtape.py b/cyberdrop_dl/crawlers/streamtape.pyindex 0950444c9..a8492df11 100644--- a/cyberdrop_dl/crawlers/streamtape.py+++ b/cyberdrop_dl/crawlers/streamtape.py@@ -1,6 +1,7 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+import asyncio+from typing import TYPE_CHECKING, ClassVar, finalfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -13,6 +14,7 @@ if TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem+@finalclass Selector:BAIT_LINK = "#ideoooolink"JS_TOKEN = "script:-soup-contains(ideoooolink)" # noqa: S105@@ -20,8 +22,10 @@ class Selector:class StreamtapeCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Videos": "/v/<video_id>",- "Player": "/e/<video_id>",+ "Videos": (+ "/v/<video_id>",+ "/e/<video_id>",+ )}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://streamtape.com")DOMAIN: ClassVar[str] = "streamtape.com"@@ -38,18 +42,23 @@ class StreamtapeCrawler(Crawler):async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:scrape_item.url = self.PRIMARY_URL / "v" / video_idif await self.check_complete_from_referer(scrape_item.url):- return None+ returnsoup = await self.request_soup(scrape_item.url)- link = _extract_download_link(soup)+ link = await asyncio.to_thread(_extract_dl_url, soup)name = open_graph.title(soup)- filename, ext = self.get_filename_and_ext(name)- return await self.handle_file(- scrape_item.url, scrape_item, name, ext, debrid_link=link, custom_filename=filename+ _, ext = self.get_filename_and_ext(name)+ await self.handle_file(+ scrape_item.url,+ scrape_item,+ name,+ ext,+ debrid_link=link,+ custom_filename=self.create_custom_filename(name, ext, file_id=video_id),)-def _extract_download_link(soup: BeautifulSoup) -> AbsoluteHttpURL:+def _extract_dl_url(soup: BeautifulSoup) -> AbsoluteHttpURL:script = css.select_text(soup, Selector.JS_TOKEN)token = extr_text(script, "&token=", "'")bait_url = css.select_text(soup, Selector.BAIT_LINK)diff --git a/cyberdrop_dl/utils/js_unpacker.py b/cyberdrop_dl/utils/js_unpacker.pyindex 479f5059b..014657119 100644--- a/cyberdrop_dl/utils/js_unpacker.py+++ b/cyberdrop_dl/utils/js_unpacker.py@@ -25,15 +25,23 @@ from __future__ import annotationsimport refrom typing import TYPE_CHECKING+from cyberdrop_dl.utils import css+if TYPE_CHECKING:from collections.abc import Callable+ import bs4+_ALPHABET = {62: "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",95: (" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"),}+def find(soup: bs4.Tag) -> str:+ return css.select_text(soup, "script:-soup-contains('p,a,c,k,e,d')")++def unpack(source: str) -> str:content, base, count, words_list = _parse(source)diff --git a/tests/crawlers/test_cases/lulustream.py b/tests/crawlers/test_cases/lulustream.pynew file mode 100644index 000000000..18e800c94--- /dev/null+++ b/tests/crawlers/test_cases/lulustream.py@@ -0,0 +1,20 @@+DOMAIN = "lulustream"+TEST_CASES = [+ {+ "url": "https://luluvid.com/lhciecvqv7e2",+ "results": [+ {+ "url": "https://lulustream.com/e/lhciecvqv7e2",+ "filename": "Rashmiix @rmiiix69 - Let me show you my outfit [1920p].mp4",+ "debrid_url": None,+ "original_filename": "Rashmiix @rmiiix69 - Let me show you my outfit",+ "referer": "https://luluvid.com/lhciecvqv7e2",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:Loose Files (Lulustream)",+ "thumbnail": "https://img.lulucdn.com/lhciecvqv7e2_xt.jpg",+ }+ ],+ "count": 1,+ }+]