Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: always use m3u8 (beeg)
NTFSvolume31df8dd40fe2
diff --git a/cyberdrop_dl/crawlers/beeg.py b/cyberdrop_dl/crawlers/beeg.pyindex b2b6ec33d..871f83d64 100644--- a/cyberdrop_dl/crawlers/beeg.py+++ b/cyberdrop_dl/crawlers/beeg.py@@ -1,43 +1,18 @@from __future__ import annotationsimport dataclasses-from collections.abc import Generator-from typing import TYPE_CHECKING, Any, ClassVar, Literal+from typing import TYPE_CHECKING, Any, ClassVar, Finalfrom cyberdrop_dl.clients.http import HTTPConfig-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils.dataclass import DictDataclass+from cyberdrop_dl.utils import datesfrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import Generator-from cyberdrop_dl.url_objects import ScrapeItem-_JSON_URL = AbsoluteHttpURL("https://store.externulls.com/facts/file/")-_VIDEO_CDN = AbsoluteHttpURL("https://video.beeg.com/")--[email protected](slots=True)-class Video:- title: str- created_at: str- mp4_formats: tuple[Format, ...]- hls_formats: tuple[Format, ...]--[email protected](slots=True, order=True)-class Format(DictDataclass):- codec: Literal["h264"]- quality: int- average_bandwidth: int- size: int- hdr: bool- url: AbsoluteHttpURL--@HTTPConfig(rate_limit=(4, 1))class BeegComCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {@@ -47,13 +22,16 @@ class BeegComCrawler(Crawler):)}DOMAIN: ClassVar[str] = "beeg.com"- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://beeg.com/")+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://beeg.com")++ def __post_init__(self) -> None:+ self.api: BeegAPI = BeegAPI.from_crawler(self)async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case [video_id]:- video_id = str(int(video_id.removeprefix("-")))- return await self.video(scrape_item, video_id)+ video_id = int(video_id.removeprefix("-"))+ await self.video(scrape_item, video_id)case _:raise ValueError@@ -67,55 +45,55 @@ class BeegComCrawler(Crawler):return url@error_handling_wrapper- async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:+ async def video(self, scrape_item: ScrapeItem, video_id: int) -> None:if await self.check_complete_from_referer(scrape_item.url):return- video = await self._request_video(video_id)- scrape_item.uploaded_at = self.parse_iso_date(video.created_at)- best = max(video.hls_formats)- m3u8, _ = await self.request_m3u8(best.url)- filename = self.create_custom_filename(- video.title,- ext := ".mp4",- file_id=video_id,- resolution=best.quality,- )+ video = await self.api.video(video_id)+ scrape_item.uploaded_at = video.created_at+ m3u8, info = await self.request_m3u8_playlist(video.m3u8)await self.handle_file(scrape_item.url,scrape_item,video.title,- ext,- custom_filename=filename,+ ext := ".mp4",+ custom_filename=self.create_custom_filename(+ video.title,+ ext,+ file_id=str(video.id),+ resolution=info.resolution,+ video_codec=info.codecs.video,+ audio_codec=info.codecs.audio,+ ),m3u8=m3u8,+ thumbnail=video.thumb,)- async def _request_video(self, video_id: str) -> Video:- resp: dict[str, Any] = await self.request_json(_JSON_URL / video_id)- facts: dict[str, Any] = min(resp["fc_facts"], key=lambda x: int(x["id"]))- file: dict[str, Any] = resp["file"]- return Video(- title=next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"],- created_at=facts["fc_created"],- hls_formats=tuple(_parse_hls_formats(file["hls_resources"])),- mp4_formats=tuple(_parse_mp4_formats(file["qualities"])),- )+[email protected](order=True, frozen=True, slots=True)+class Video:+ id: int+ title: str+ created_at: int+ thumb: AbsoluteHttpURL+ m3u8: AbsoluteHttpURL-def _parse_hls_formats(sources: dict[str, str]) -> Generator[Format]:- for name, uri in sources.items():- if "multi" not in name:- yield Format(- codec="h264",- quality=int(name.removeprefix("fl_cdn_")),- url=_VIDEO_CDN / uri,- average_bandwidth=0,- hdr=False,- size=0,- )+class BeegAPI(API):+ VIDEO: Final = AbsoluteHttpURL("https://video.beeg.com")+ STORE: Final = AbsoluteHttpURL("https://store.externulls.com")+ THUMBS: Final = AbsoluteHttpURL("https://thumbs.externulls.com")+ async def video(self, video_id: int) -> Video:+ resp: dict[str, Any] = await self.request_json(self.STORE / f"facts/file/{video_id}")+ facts: dict[str, Any] = min(resp["fc_facts"], key=lambda x: int(x["id"]))+ file: dict[str, Any] = resp["file"]+ thumb = next(iter(facts["fc_thumbs"]), 0)-def _parse_mp4_formats(qualities: dict[str, list[dict[str, Any]]]) -> Generator[Format]:- for sources in qualities.values():- for source in sources:- yield Format.from_dict(source, url=_VIDEO_CDN / source["url"])+ return Video(+ id=video_id,+ title=next(data for data in file["data"] if data.get("cd_file") == video_id)["cd_value"],+ created_at=int(dates.parse_iso(facts["fc_created"]).timestamp()),+ thumb=self.THUMBS / f"videos/{video_id}/{thumb}.webp",+ m3u8=self.VIDEO / file["hls_resources"]["fl_cdn_multi"],+ )diff --git a/tests/crawlers/test_cases/beeg_com.py b/tests/crawlers/test_cases/beeg_com.pyindex 3af5d3b1a..7dffc92a0 100644--- a/tests/crawlers/test_cases/beeg_com.py+++ b/tests/crawlers/test_cases/beeg_com.py@@ -5,13 +5,16 @@ TEST_CASES = ["results": [{"url": "https://beeg.com/-0825220911617085",- "referer": "https://beeg.com/-0825220911617085",- "download_folder": "re:Loose Files (Beeg.com)",- "filename": "Blonde in Sexy Dress Takes Deep Anal Dildo [825220911617085][1080p].mp4",+ "filename": "Blonde in Sexy Dress Takes Deep Anal Dildo [825220911617085][avc1][mp4a][1080p].mp4",+ "debrid_url": None,"original_filename": "Blonde in Sexy Dress Takes Deep Anal Dildo",+ "referer": "https://beeg.com/-0825220911617085","album_id": None,"uploaded_at": 1773100800,+ "download_folder": "re:Loose Files (Beeg.com)",+ "thumbnail": "https://thumbs.externulls.com/videos/825220911617085/146.webp",}],- }+ "count": 1,+ },]