Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: parse mp4 formats (beeg) (#106)
NTFSvolume8d571b712d5b
diff --git a/cyberdrop_dl/crawlers/beeg.py b/cyberdrop_dl/crawlers/beeg.pyindex f7f044ca3..f991b0738 100644--- a/cyberdrop_dl/crawlers/beeg.py+++ b/cyberdrop_dl/crawlers/beeg.py@@ -1,22 +1,38 @@from __future__ import annotations-from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple+import dataclasses+from collections.abc import Generator+from typing import TYPE_CHECKING, Any, ClassVar, Literal-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths-from cyberdrop_dl.data_structures.mediaprops import Resolution+from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils.utilities import DictDataclass, error_handling_wrapperif TYPE_CHECKING:+ from collections.abc import Generator+from cyberdrop_dl.data_structures.url_objects import ScrapeItem-PRIMARY_URL = AbsoluteHttpURL("https://beeg.com/")-JSON_URL = AbsoluteHttpURL("https://store.externulls.com/facts/file/")-M3U8_URL = AbsoluteHttpURL("https://video.beeg.com/")+_JSON_URL = AbsoluteHttpURL("https://store.externulls.com/facts/file/")+_VIDEO_CDN = AbsoluteHttpURL("https://video.beeg.com/")++[email protected](slots=True)+class Video:+ title: str+ created_at: str+ mp4_formats: tuple[Format, ...]+ hls_formats: tuple[Format, ...]-class Format(NamedTuple):- resolution: Resolution[email protected](slots=True, order=True)+class Format(DictDataclass):+ codec: Literal["h264"]+ quality: int+ average_bandwidth: int+ size: int+ hdr: boolurl: AbsoluteHttpURL@@ -28,50 +44,76 @@ class BeegComCrawler(Crawler):)}DOMAIN: ClassVar[str] = "beeg.com"- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL- _RATE_LIMIT = 4, 1+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://beeg.com/")+ _RATE_LIMIT: ClassVar[RateLimit] = 4, 1async def fetch(self, scrape_item: ScrapeItem) -> None:- if video_id := get_video_id(scrape_item.url):- return await self.video(scrape_item, video_id)- raise ValueError-+ match scrape_item.url.parts[1:]:+ case [video_id]:+ video_id = str(int(video_id.removeprefix("-")))+ return await self.video(scrape_item, video_id)+ case _:+ raise ValueError++ @classmethod+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ url = super().transform_url(url)+ match url.parts[1:]:+ case ["video", video_id]:+ return url.origin() / video_id+ case _:+ return url++ @error_handling_wrapperasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- canonical_url = PRIMARY_URL / video_id- if await self.check_complete_from_referer(canonical_url):+ if await self.check_complete_from_referer(scrape_item.url):return- scrape_item.url = canonical_url- json_resp: dict[str, Any] = await self.request_json(JSON_URL / video_id)- facts: dict[str, Any] = min(json_resp["fc_facts"], key=lambda x: int(x["id"]))- file: dict[str, Any] = json_resp["file"]- title: str = next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"]- best_format = get_best_format(file["hls_resources"])- scrape_item.uploaded_at = self.parse_iso_date(facts.get("fc_created", ""))- m3u8 = await self.get_m3u8_from_index_url(best_format.url)- filename, ext = self.get_filename_and_ext(best_format.url.name)- custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=best_format.resolution)- await self.handle_file(canonical_url, scrape_item, filename, ext, custom_filename=custom_filename, m3u8=m3u8)---def get_video_id(url: AbsoluteHttpURL) -> str | None:- # https://beeg.com/-0983946056129650"- # https://beeg.com/1277207756"-- index = 2 if "video" in url.parts else 1- try:- return str(int(url.parts[index].removeprefix("-")))- except Exception:- return---def get_best_format(sources: dict[str, str]) -> Format:- def parse_sources():- for name, uri in sources.items():- try:- yield Resolution.parse(name.removeprefix("fl_cdn_")), uri- except ValueError:- continue-- resolution, uri = max(parse_sources())- return Format(resolution, M3U8_URL / uri)+ video = await self._request_video(video_id)+ scrape_item.uploaded_at = self.parse_iso_date(video.created_at)+ best = max(video.hls_formats)+ m3u8, _ = await self.request_m3u8(best.url)+ filename = self.create_custom_filename(+ video.title,+ ext := ".mp4",+ file_id=video_id,+ resolution=best.quality,+ )+ await self.handle_file(+ scrape_item.url,+ scrape_item,+ video.title,+ ext,+ custom_filename=filename,+ m3u8=m3u8,+ )++ async def _request_video(self, video_id: str) -> Video:+ resp: dict[str, Any] = await self.request_json(_JSON_URL / video_id)+ facts: dict[str, Any] = min(resp["fc_facts"], key=lambda x: int(x["id"]))+ file: dict[str, Any] = resp["file"]+ return Video(+ title=next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"],+ created_at=facts["fc_created"],+ hls_formats=tuple(_parse_hls_formats(file["hls_resources"])),+ mp4_formats=tuple(_parse_mp4_formats(file["qualities"])),+ )+++def _parse_hls_formats(sources: dict[str, str]) -> Generator[Format]:+ for name, uri in sources.items():+ if "multi" not in name:+ yield Format(+ codec="h264",+ quality=int(name.removeprefix("fl_cdn_")),+ url=_VIDEO_CDN / uri,+ average_bandwidth=0,+ hdr=False,+ size=0,+ )+++def _parse_mp4_formats(qualites: dict[str, list[dict[str, Any]]]) -> Generator[Format]:+ for _, sources in qualites.items():+ for source in sources:+ yield Format.from_dict(source, url=_VIDEO_CDN / source["url"])diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex b40a88086..ee29704f6 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -88,7 +88,9 @@ class DictDataclass(Dataclass, Protocol):return {name: data.get(name) for name in _fields(cls)}@classmethod- def from_dict(cls, data: dict[str, Any], /) -> Self:+ def from_dict(cls, data: dict[str, Any], /, **overrides: Any) -> Self:+ if overrides:+ data.update(overrides)return cls(**cls.filter_dict(data))diff --git a/tests/crawlers/test_cases/beeg.py b/tests/crawlers/test_cases/beeg.pynew file mode 100644index 000000000..62252afd0--- /dev/null+++ b/tests/crawlers/test_cases/beeg.py@@ -0,0 +1,17 @@+DOMAIN = "beeg.com"+TEST_CASES = [+ (+ "https://beeg.com/-0825220911617085",+ [+ {+ "url": "https://beeg.com/-0825220911617085",+ "referer": "https://beeg.com/-0825220911617085",+ "download_folder": "re:Loose Files (Beeg.com)",+ "filename": "Blonde in Sexy Dress Takes Deep Anal Dildo [825220911617085][1080p].mp4",+ "original_filename": "Blonde in Sexy Dress Takes Deep Anal Dildo",+ "album_id": None,+ "datetime": 1773100800,+ }+ ],+ ),+]