Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: parse mp4 formats (beeg) (#106)

NTFSvolume8d571b712d5b
diff --git a/cyberdrop_dl/crawlers/beeg.py b/cyberdrop_dl/crawlers/beeg.py
index f7f044ca3..f991b0738 100644
--- a/cyberdrop_dl/crawlers/beeg.py
+++ b/cyberdrop_dl/crawlers/beeg.py
@@ -1,22 +1,38 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple
+import dataclasses
+from collections.abc import Generator
+from typing import TYPE_CHECKING, Any, ClassVar, Literal
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
-from cyberdrop_dl.data_structures.mediaprops import Resolution
+from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils.utilities import DictDataclass, error_handling_wrapper
if TYPE_CHECKING:
+ from collections.abc import Generator
+
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-PRIMARY_URL = AbsoluteHttpURL("https://beeg.com/")
-JSON_URL = AbsoluteHttpURL("https://store.externulls.com/facts/file/")
-M3U8_URL = AbsoluteHttpURL("https://video.beeg.com/")
+_JSON_URL = AbsoluteHttpURL("https://store.externulls.com/facts/file/")
+_VIDEO_CDN = AbsoluteHttpURL("https://video.beeg.com/")
+
+
[email protected](slots=True)
+class Video:
+ title: str
+ created_at: str
+ mp4_formats: tuple[Format, ...]
+ hls_formats: tuple[Format, ...]
-class Format(NamedTuple):
- resolution: Resolution
[email protected](slots=True, order=True)
+class Format(DictDataclass):
+ codec: Literal["h264"]
+ quality: int
+ average_bandwidth: int
+ size: int
+ hdr: bool
url: AbsoluteHttpURL
@@ -28,50 +44,76 @@ class BeegComCrawler(Crawler):
)
}
DOMAIN: ClassVar[str] = "beeg.com"
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
- _RATE_LIMIT = 4, 1
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://beeg.com/")
+ _RATE_LIMIT: ClassVar[RateLimit] = 4, 1
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if video_id := get_video_id(scrape_item.url):
- return await self.video(scrape_item, video_id)
- raise ValueError
-
+ match scrape_item.url.parts[1:]:
+ case [video_id]:
+ video_id = str(int(video_id.removeprefix("-")))
+ return await self.video(scrape_item, video_id)
+ case _:
+ raise ValueError
+
+ @classmethod
+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ url = super().transform_url(url)
+ match url.parts[1:]:
+ case ["video", video_id]:
+ return url.origin() / video_id
+ case _:
+ return url
+
+ @error_handling_wrapper
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- canonical_url = PRIMARY_URL / video_id
- if await self.check_complete_from_referer(canonical_url):
+ if await self.check_complete_from_referer(scrape_item.url):
return
- scrape_item.url = canonical_url
- json_resp: dict[str, Any] = await self.request_json(JSON_URL / video_id)
- facts: dict[str, Any] = min(json_resp["fc_facts"], key=lambda x: int(x["id"]))
- file: dict[str, Any] = json_resp["file"]
- title: str = next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"]
- best_format = get_best_format(file["hls_resources"])
- scrape_item.uploaded_at = self.parse_iso_date(facts.get("fc_created", ""))
- m3u8 = await self.get_m3u8_from_index_url(best_format.url)
- filename, ext = self.get_filename_and_ext(best_format.url.name)
- custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=best_format.resolution)
- await self.handle_file(canonical_url, scrape_item, filename, ext, custom_filename=custom_filename, m3u8=m3u8)
-
-
-def get_video_id(url: AbsoluteHttpURL) -> str | None:
- # https://beeg.com/-0983946056129650"
- # https://beeg.com/1277207756"
-
- index = 2 if "video" in url.parts else 1
- try:
- return str(int(url.parts[index].removeprefix("-")))
- except Exception:
- return
-
-
-def get_best_format(sources: dict[str, str]) -> Format:
- def parse_sources():
- for name, uri in sources.items():
- try:
- yield Resolution.parse(name.removeprefix("fl_cdn_")), uri
- except ValueError:
- continue
-
- resolution, uri = max(parse_sources())
- return Format(resolution, M3U8_URL / uri)
+ video = await self._request_video(video_id)
+ scrape_item.uploaded_at = self.parse_iso_date(video.created_at)
+ best = max(video.hls_formats)
+ m3u8, _ = await self.request_m3u8(best.url)
+ filename = self.create_custom_filename(
+ video.title,
+ ext := ".mp4",
+ file_id=video_id,
+ resolution=best.quality,
+ )
+ await self.handle_file(
+ scrape_item.url,
+ scrape_item,
+ video.title,
+ ext,
+ custom_filename=filename,
+ m3u8=m3u8,
+ )
+
+ async def _request_video(self, video_id: str) -> Video:
+ resp: dict[str, Any] = await self.request_json(_JSON_URL / video_id)
+ facts: dict[str, Any] = min(resp["fc_facts"], key=lambda x: int(x["id"]))
+ file: dict[str, Any] = resp["file"]
+ return Video(
+ title=next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"],
+ created_at=facts["fc_created"],
+ hls_formats=tuple(_parse_hls_formats(file["hls_resources"])),
+ mp4_formats=tuple(_parse_mp4_formats(file["qualities"])),
+ )
+
+
+def _parse_hls_formats(sources: dict[str, str]) -> Generator[Format]:
+ for name, uri in sources.items():
+ if "multi" not in name:
+ yield Format(
+ codec="h264",
+ quality=int(name.removeprefix("fl_cdn_")),
+ url=_VIDEO_CDN / uri,
+ average_bandwidth=0,
+ hdr=False,
+ size=0,
+ )
+
+
+def _parse_mp4_formats(qualites: dict[str, list[dict[str, Any]]]) -> Generator[Format]:
+ for _, sources in qualites.items():
+ for source in sources:
+ yield Format.from_dict(source, url=_VIDEO_CDN / source["url"])
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index b40a88086..ee29704f6 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -88,7 +88,9 @@ class DictDataclass(Dataclass, Protocol):
return {name: data.get(name) for name in _fields(cls)}
@classmethod
- def from_dict(cls, data: dict[str, Any], /) -> Self:
+ def from_dict(cls, data: dict[str, Any], /, **overrides: Any) -> Self:
+ if overrides:
+ data.update(overrides)
return cls(**cls.filter_dict(data))
diff --git a/tests/crawlers/test_cases/beeg.py b/tests/crawlers/test_cases/beeg.py
new file mode 100644
index 000000000..62252afd0
--- /dev/null
+++ b/tests/crawlers/test_cases/beeg.py
@@ -0,0 +1,17 @@
+DOMAIN = "beeg.com"
+TEST_CASES = [
+ (
+ "https://beeg.com/-0825220911617085",
+ [
+ {
+ "url": "https://beeg.com/-0825220911617085",
+ "referer": "https://beeg.com/-0825220911617085",
+ "download_folder": "re:Loose Files (Beeg.com)",
+ "filename": "Blonde in Sexy Dress Takes Deep Anal Dildo [825220911617085][1080p].mp4",
+ "original_filename": "Blonde in Sexy Dress Takes Deep Anal Dildo",
+ "album_id": None,
+ "datetime": 1773100800,
+ }
+ ],
+ ),
+]