Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add lulustream support

NTFSvolumed80fb1dbfd29
diff --git a/cyberdrop_dl/crawlers/lulustream.py b/cyberdrop_dl/crawlers/lulustream.py
new file mode 100644
index 000000000..3f67519e2
--- /dev/null
+++ b/cyberdrop_dl/crawlers/lulustream.py
@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl import aio
+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedDomains, SupportedPaths
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import TextExtractor, css, js_unpacker
+from cyberdrop_dl.utils.errors import error_handling_wrapper
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.url_objects import ScrapeItem
+
+
+class LuluStreamCrawler(Crawler):
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (
+ "lulustream",
+ "luluvdo.com",
+ "lulu.st",
+ "luluvid.com",
+ "cdn1.site",
+ "streamhihi.com",
+ "luluvdoo.com",
+ )
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Video": (
+ "/e/<video_id>",
+ "/d/<video_id>",
+ "/<video_id>",
+ ),
+ }
+ DOMAIN: ClassVar[str] = "lulustream"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://lulustream.com")
+
+ def __post_init__(self) -> None:
+ self.api: LuluStreamAPI = LuluStreamAPI.from_crawler(self)
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["e" | "d", video_id] | [video_id]:
+ await self.video(scrape_item, video_id)
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
+ embed_url = self.PRIMARY_URL / "e" / video_id
+ if await self.check_complete(embed_url):
+ return
+
+ video = await self.api.video(video_id)
+ m3u8, info = await self.request_m3u8_playlist(video.m3u8)
+ await self.handle_file(
+ embed_url,
+ scrape_item,
+ video.title,
+ ext := ".mp4",
+ m3u8=m3u8,
+ custom_filename=self.create_custom_filename(video.title, ext, resolution=info.resolution),
+ thumbnail=video.thumb,
+ )
+
+
[email protected](slots=True, frozen=True)
+class Video:
+ id: str
+ title: str
+ m3u8: AbsoluteHttpURL
+ thumb: AbsoluteHttpURL
+
+
+class LuluStreamAPI(API):
+ async def video(self, video_id: str) -> Video:
+ embed, web = await aio.gather(
+ self.request_soup(self.origin / "e" / video_id),
+ self.request_soup(self.origin / "d" / video_id),
+ fail_fast=False,
+ )
+ player = js_unpacker.unpack(js_unpacker.find(embed))
+ extr = TextExtractor(player)
+ return Video(
+ id=video_id,
+ title=css.select_text(web, "h1"),
+ m3u8=self.parse_url(extr('{sources:[{file:"', '"')),
+ thumb=self.parse_url(extr('image:"', '"')),
+ )
diff --git a/cyberdrop_dl/crawlers/streamtape.py b/cyberdrop_dl/crawlers/streamtape.py
index 0950444c9..a8492df11 100644
--- a/cyberdrop_dl/crawlers/streamtape.py
+++ b/cyberdrop_dl/crawlers/streamtape.py
@@ -1,6 +1,7 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+import asyncio
+from typing import TYPE_CHECKING, ClassVar, final
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -13,6 +14,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
+@final
class Selector:
BAIT_LINK = "#ideoooolink"
JS_TOKEN = "script:-soup-contains(ideoooolink)" # noqa: S105
@@ -20,8 +22,10 @@ class Selector:
class StreamtapeCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Videos": "/v/<video_id>",
- "Player": "/e/<video_id>",
+ "Videos": (
+ "/v/<video_id>",
+ "/e/<video_id>",
+ )
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://streamtape.com")
DOMAIN: ClassVar[str] = "streamtape.com"
@@ -38,18 +42,23 @@ class StreamtapeCrawler(Crawler):
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
scrape_item.url = self.PRIMARY_URL / "v" / video_id
if await self.check_complete_from_referer(scrape_item.url):
- return None
+ return
soup = await self.request_soup(scrape_item.url)
- link = _extract_download_link(soup)
+ link = await asyncio.to_thread(_extract_dl_url, soup)
name = open_graph.title(soup)
- filename, ext = self.get_filename_and_ext(name)
- return await self.handle_file(
- scrape_item.url, scrape_item, name, ext, debrid_link=link, custom_filename=filename
+ _, ext = self.get_filename_and_ext(name)
+ await self.handle_file(
+ scrape_item.url,
+ scrape_item,
+ name,
+ ext,
+ debrid_link=link,
+ custom_filename=self.create_custom_filename(name, ext, file_id=video_id),
)
-def _extract_download_link(soup: BeautifulSoup) -> AbsoluteHttpURL:
+def _extract_dl_url(soup: BeautifulSoup) -> AbsoluteHttpURL:
script = css.select_text(soup, Selector.JS_TOKEN)
token = extr_text(script, "&token=", "'")
bait_url = css.select_text(soup, Selector.BAIT_LINK)
diff --git a/cyberdrop_dl/utils/js_unpacker.py b/cyberdrop_dl/utils/js_unpacker.py
index 479f5059b..014657119 100644
--- a/cyberdrop_dl/utils/js_unpacker.py
+++ b/cyberdrop_dl/utils/js_unpacker.py
@@ -25,15 +25,23 @@ from __future__ import annotations
import re
from typing import TYPE_CHECKING
+from cyberdrop_dl.utils import css
+
if TYPE_CHECKING:
from collections.abc import Callable
+ import bs4
+
_ALPHABET = {
62: "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
95: (" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"),
}
+def find(soup: bs4.Tag) -> str:
+ return css.select_text(soup, "script:-soup-contains('p,a,c,k,e,d')")
+
+
def unpack(source: str) -> str:
content, base, count, words_list = _parse(source)
diff --git a/tests/crawlers/test_cases/lulustream.py b/tests/crawlers/test_cases/lulustream.py
new file mode 100644
index 000000000..18e800c94
--- /dev/null
+++ b/tests/crawlers/test_cases/lulustream.py
@@ -0,0 +1,20 @@
+DOMAIN = "lulustream"
+TEST_CASES = [
+ {
+ "url": "https://luluvid.com/lhciecvqv7e2",
+ "results": [
+ {
+ "url": "https://lulustream.com/e/lhciecvqv7e2",
+ "filename": "Rashmiix @rmiiix69 - Let me show you my outfit [1920p].mp4",
+ "debrid_url": None,
+ "original_filename": "Rashmiix @rmiiix69 - Let me show you my outfit",
+ "referer": "https://luluvid.com/lhciecvqv7e2",
+ "album_id": None,
+ "uploaded_at": None,
+ "download_folder": "re:Loose Files (Lulustream)",
+ "thumbnail": "https://img.lulucdn.com/lhciecvqv7e2_xt.jpg",
+ }
+ ],
+ "count": 1,
+ }
+]