Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add tiktikstream support
NTFSvolume695ee19bdc88
diff --git a/cyberdrop_dl/crawlers/lulustream.py b/cyberdrop_dl/crawlers/lulustream.pyindex 515efb7e9..6ed68ff69 100644--- a/cyberdrop_dl/crawlers/lulustream.py+++ b/cyberdrop_dl/crawlers/lulustream.py@@ -16,7 +16,10 @@ if TYPE_CHECKING:class LuluStreamCrawler(Crawler):SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ("lulustream",- "luluvdo.com",+ "luluvdo",+ "luluvido",+ "luluvd",+ "luluvid","lulu.st","luluvid.com","cdn1.site",diff --git a/cyberdrop_dl/crawlers/tiktikstream.py b/cyberdrop_dl/crawlers/tiktikstream.pynew file mode 100644index 000000000..487120056--- /dev/null+++ b/cyberdrop_dl/crawlers/tiktikstream.py@@ -0,0 +1,100 @@+from __future__ import annotations++import dataclasses+from typing import TYPE_CHECKING, ClassVar, override++from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths+from cyberdrop_dl.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import next_js, traversal+from cyberdrop_dl.utils.errors import error_handling_wrapper++if TYPE_CHECKING:+ from cyberdrop_dl.url_objects import ScrapeItem+++class TiktikStreamCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Video": (+ "/embed/<video_slug>",+ "/v/<video_slug>",+ ),+ }+ DOMAIN: ClassVar[str] = "tiktikstream"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://tiktikstream.com")++ @classmethod+ @override+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ url = super().transform_url(url).with_query(None)+ match url.parts[1:]:+ case ["embed", video_id]:+ return url.origin() / "v" / video_id+ case _:+ return url++ def __post_init__(self) -> None:+ self.api: TiktikStreamAPI = TiktikStreamAPI.from_crawler(self)++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["v", video_slug]:+ await self.video(scrape_item, video_slug)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def video(self, scrape_item: ScrapeItem, slug: str) -> None:+ web_url = self.PRIMARY_URL / "v" / slug+ if await self.check_complete(web_url):+ return++ video = await self.api.video(slug)+ m3u8, info = await self.request_m3u8_playlist(+ video.m3u8,+ headers={"Authorization": f"Bearer {video.token}", "Referer": str(web_url), "Origin": str(self.origin)},+ keep_query=True,+ )+ await self.handle_file(+ web_url,+ scrape_item,+ video.title,+ ext := ".mp4",+ m3u8=m3u8,+ custom_filename=self.create_custom_filename(video.title, ext, file_id=slug, resolution=info.resolution),+ thumbnail=video.thumb,+ )++[email protected](slots=True, frozen=True)+class Video:+ id: str+ slug: str+ title: str+ token: str+ m3u8: AbsoluteHttpURL+ thumb: AbsoluteHttpURL+++class TiktikStreamAPI(API):+ async def video(self, slug: str) -> Video:+ web_url = self.origin / "v" / slug+ soup = await self.request_soup(web_url)+ flight = next_js.extract(soup)+ _, video = traversal.find_obj(flight, "videoId", "title", "suggestions")+ video_uuid = video["videoId"]+ api_url = self.origin / "api/playback/sessions"+ data = await self.request_json(+ api_url,+ "POST",+ headers={"Referer": str(web_url), "Origin": str(self.origin)},+ json={"videoId": video_uuid, "referrer": str(web_url), "embedMode": False},+ )++ return Video(+ id=video_uuid,+ slug=slug,+ title=video["title"],+ token=data["token"],+ m3u8=self.parse_url(data["streamUrl"]),+ thumb=self.origin / "api/videos" / video_uuid / "thumbnail",+ )