Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: always use m3u8 (beeg)

NTFSvolume31df8dd40fe2
diff --git a/cyberdrop_dl/crawlers/beeg.py b/cyberdrop_dl/crawlers/beeg.py
index b2b6ec33d..871f83d64 100644
--- a/cyberdrop_dl/crawlers/beeg.py
+++ b/cyberdrop_dl/crawlers/beeg.py
@@ -1,43 +1,18 @@
from __future__ import annotations
import dataclasses
-from collections.abc import Generator
-from typing import TYPE_CHECKING, Any, ClassVar, Literal
+from typing import TYPE_CHECKING, Any, ClassVar, Final
from cyberdrop_dl.clients.http import HTTPConfig
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils.dataclass import DictDataclass
+from cyberdrop_dl.utils import dates
from cyberdrop_dl.utils.errors import error_handling_wrapper
if TYPE_CHECKING:
- from collections.abc import Generator
-
from cyberdrop_dl.url_objects import ScrapeItem
-_JSON_URL = AbsoluteHttpURL("https://store.externulls.com/facts/file/")
-_VIDEO_CDN = AbsoluteHttpURL("https://video.beeg.com/")
-
-
[email protected](slots=True)
-class Video:
- title: str
- created_at: str
- mp4_formats: tuple[Format, ...]
- hls_formats: tuple[Format, ...]
-
-
[email protected](slots=True, order=True)
-class Format(DictDataclass):
- codec: Literal["h264"]
- quality: int
- average_bandwidth: int
- size: int
- hdr: bool
- url: AbsoluteHttpURL
-
-
@HTTPConfig(rate_limit=(4, 1))
class BeegComCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
@@ -47,13 +22,16 @@ class BeegComCrawler(Crawler):
)
}
DOMAIN: ClassVar[str] = "beeg.com"
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://beeg.com/")
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://beeg.com")
+
+ def __post_init__(self) -> None:
+ self.api: BeegAPI = BeegAPI.from_crawler(self)
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [video_id]:
- video_id = str(int(video_id.removeprefix("-")))
- return await self.video(scrape_item, video_id)
+ video_id = int(video_id.removeprefix("-"))
+ await self.video(scrape_item, video_id)
case _:
raise ValueError
@@ -67,55 +45,55 @@ class BeegComCrawler(Crawler):
return url
@error_handling_wrapper
- async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
+ async def video(self, scrape_item: ScrapeItem, video_id: int) -> None:
if await self.check_complete_from_referer(scrape_item.url):
return
- video = await self._request_video(video_id)
- scrape_item.uploaded_at = self.parse_iso_date(video.created_at)
- best = max(video.hls_formats)
- m3u8, _ = await self.request_m3u8(best.url)
- filename = self.create_custom_filename(
- video.title,
- ext := ".mp4",
- file_id=video_id,
- resolution=best.quality,
- )
+ video = await self.api.video(video_id)
+ scrape_item.uploaded_at = video.created_at
+ m3u8, info = await self.request_m3u8_playlist(video.m3u8)
await self.handle_file(
scrape_item.url,
scrape_item,
video.title,
- ext,
- custom_filename=filename,
+ ext := ".mp4",
+ custom_filename=self.create_custom_filename(
+ video.title,
+ ext,
+ file_id=str(video.id),
+ resolution=info.resolution,
+ video_codec=info.codecs.video,
+ audio_codec=info.codecs.audio,
+ ),
m3u8=m3u8,
+ thumbnail=video.thumb,
)
- async def _request_video(self, video_id: str) -> Video:
- resp: dict[str, Any] = await self.request_json(_JSON_URL / video_id)
- facts: dict[str, Any] = min(resp["fc_facts"], key=lambda x: int(x["id"]))
- file: dict[str, Any] = resp["file"]
- return Video(
- title=next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"],
- created_at=facts["fc_created"],
- hls_formats=tuple(_parse_hls_formats(file["hls_resources"])),
- mp4_formats=tuple(_parse_mp4_formats(file["qualities"])),
- )
+
[email protected](order=True, frozen=True, slots=True)
+class Video:
+ id: int
+ title: str
+ created_at: int
+ thumb: AbsoluteHttpURL
+ m3u8: AbsoluteHttpURL
-def _parse_hls_formats(sources: dict[str, str]) -> Generator[Format]:
- for name, uri in sources.items():
- if "multi" not in name:
- yield Format(
- codec="h264",
- quality=int(name.removeprefix("fl_cdn_")),
- url=_VIDEO_CDN / uri,
- average_bandwidth=0,
- hdr=False,
- size=0,
- )
+class BeegAPI(API):
+ VIDEO: Final = AbsoluteHttpURL("https://video.beeg.com")
+ STORE: Final = AbsoluteHttpURL("https://store.externulls.com")
+ THUMBS: Final = AbsoluteHttpURL("https://thumbs.externulls.com")
+ async def video(self, video_id: int) -> Video:
+ resp: dict[str, Any] = await self.request_json(self.STORE / f"facts/file/{video_id}")
+ facts: dict[str, Any] = min(resp["fc_facts"], key=lambda x: int(x["id"]))
+ file: dict[str, Any] = resp["file"]
+ thumb = next(iter(facts["fc_thumbs"]), 0)
-def _parse_mp4_formats(qualities: dict[str, list[dict[str, Any]]]) -> Generator[Format]:
- for sources in qualities.values():
- for source in sources:
- yield Format.from_dict(source, url=_VIDEO_CDN / source["url"])
+ return Video(
+ id=video_id,
+ title=next(data for data in file["data"] if data.get("cd_file") == video_id)["cd_value"],
+ created_at=int(dates.parse_iso(facts["fc_created"]).timestamp()),
+ thumb=self.THUMBS / f"videos/{video_id}/{thumb}.webp",
+ m3u8=self.VIDEO / file["hls_resources"]["fl_cdn_multi"],
+ )
diff --git a/tests/crawlers/test_cases/beeg_com.py b/tests/crawlers/test_cases/beeg_com.py
index 3af5d3b1a..7dffc92a0 100644
--- a/tests/crawlers/test_cases/beeg_com.py
+++ b/tests/crawlers/test_cases/beeg_com.py
@@ -5,13 +5,16 @@ TEST_CASES = [
"results": [
{
"url": "https://beeg.com/-0825220911617085",
- "referer": "https://beeg.com/-0825220911617085",
- "download_folder": "re:Loose Files (Beeg.com)",
- "filename": "Blonde in Sexy Dress Takes Deep Anal Dildo [825220911617085][1080p].mp4",
+ "filename": "Blonde in Sexy Dress Takes Deep Anal Dildo [825220911617085][avc1][mp4a][1080p].mp4",
+ "debrid_url": None,
"original_filename": "Blonde in Sexy Dress Takes Deep Anal Dildo",
+ "referer": "https://beeg.com/-0825220911617085",
"album_id": None,
"uploaded_at": 1773100800,
+ "download_folder": "re:Loose Files (Beeg.com)",
+ "thumbnail": "https://thumbs.externulls.com/videos/825220911617085/146.webp",
}
],
- }
+ "count": 1,
+ },
]