Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: properly caching downloaded videos urls (beeg) (#1128)
Barbarella6666666b4bcd9d7aa18
diff --git a/cyberdrop_dl/crawlers/beeg.py b/cyberdrop_dl/crawlers/beeg.pyindex f7d2fb7cc..758f84a1f 100644--- a/cyberdrop_dl/crawlers/beeg.py+++ b/cyberdrop_dl/crawlers/beeg.py@@ -6,7 +6,7 @@ from aiolimiter import AsyncLimiterfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils.m3u8 import M3U8, M3U8Media+from cyberdrop_dl.utils.m3u8 import M3U8Mediaif TYPE_CHECKING:from collections.abc import Iterable@@ -43,35 +43,36 @@ class BeegComCrawler(Crawler):self.request_limiter = AsyncLimiter(4, 1)async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:+ canonical_url = PRIMARY_URL / video_id+ if await self.check_complete_from_referer(canonical_url):+ return++ scrape_item.url = canonical_urlasync with self.request_limiter:json_resp: dict[str, Any] = await self.client.get_json(self.DOMAIN, JSON_URL / video_id)facts: dict[str, Any] = min(json_resp["fc_facts"], key=lambda x: int(x["id"]))file: dict[str, Any] = json_resp["file"]- title = next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"]+ title: str = next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"]best_format = get_best_format(file["hls_resources"])- scrape_item.possible_datetime = self.parse_date(facts.get("fc_created", ""))+ scrape_item.possible_datetime = self.parse_iso_date(facts.get("fc_created", ""))m3u8_media = M3U8Media(await self._get_m3u8(best_format.url))filename, ext = self.get_filename_and_ext(best_format.url.name)custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=best_format.heigth)await self.handle_file(- best_format.url, scrape_item, filename, ext, custom_filename=custom_filename, m3u8_media=m3u8_media+ canonical_url, scrape_item, filename, ext, custom_filename=custom_filename, m3u8_media=m3u8_media)- async def _get_m3u8(self, url: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> M3U8:- headers = headers or {}- async with self.request_limiter:- content = await self.client.get_text(self.DOMAIN, url, headers)- return M3U8(content, url.parent)-def get_video_id(url: AbsoluteHttpURL) -> str | None:# https://beeg.com/-0983946056129650"# https://beeg.com/1277207756"index = 2 if "video" in url.parts else 1- if len(url.parts) > index and (video_id := url.parts[index].removeprefix("-")).isdigit():- return video_id+ try:+ return str(int(url.parts[index].removeprefix("-")))+ except Exception:+ returndef get_best_format(sources: dict[str, str]) -> Format: