Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: properly caching downloaded videos urls (beeg) (#1128)

Barbarella6666666b4bcd9d7aa18
diff --git a/cyberdrop_dl/crawlers/beeg.py b/cyberdrop_dl/crawlers/beeg.py
index f7d2fb7cc..758f84a1f 100644
--- a/cyberdrop_dl/crawlers/beeg.py
+++ b/cyberdrop_dl/crawlers/beeg.py
@@ -6,7 +6,7 @@ from aiolimiter import AsyncLimiter
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils.m3u8 import M3U8, M3U8Media
+from cyberdrop_dl.utils.m3u8 import M3U8Media
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -43,35 +43,36 @@ class BeegComCrawler(Crawler):
self.request_limiter = AsyncLimiter(4, 1)
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
+ canonical_url = PRIMARY_URL / video_id
+ if await self.check_complete_from_referer(canonical_url):
+ return
+
+ scrape_item.url = canonical_url
async with self.request_limiter:
json_resp: dict[str, Any] = await self.client.get_json(self.DOMAIN, JSON_URL / video_id)
facts: dict[str, Any] = min(json_resp["fc_facts"], key=lambda x: int(x["id"]))
file: dict[str, Any] = json_resp["file"]
- title = next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"]
+ title: str = next(data for data in file["data"] if data.get("cd_column") == "sf_name")["cd_value"]
best_format = get_best_format(file["hls_resources"])
- scrape_item.possible_datetime = self.parse_date(facts.get("fc_created", ""))
+ scrape_item.possible_datetime = self.parse_iso_date(facts.get("fc_created", ""))
m3u8_media = M3U8Media(await self._get_m3u8(best_format.url))
filename, ext = self.get_filename_and_ext(best_format.url.name)
custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=best_format.heigth)
await self.handle_file(
- best_format.url, scrape_item, filename, ext, custom_filename=custom_filename, m3u8_media=m3u8_media
+ canonical_url, scrape_item, filename, ext, custom_filename=custom_filename, m3u8_media=m3u8_media
)
- async def _get_m3u8(self, url: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> M3U8:
- headers = headers or {}
- async with self.request_limiter:
- content = await self.client.get_text(self.DOMAIN, url, headers)
- return M3U8(content, url.parent)
-
def get_video_id(url: AbsoluteHttpURL) -> str | None:
# https://beeg.com/-0983946056129650"
# https://beeg.com/1277207756"
index = 2 if "video" in url.parts else 1
- if len(url.parts) > index and (video_id := url.parts[index].removeprefix("-")).isdigit():
- return video_id
+ try:
+ return str(int(url.parts[index].removeprefix("-")))
+ except Exception:
+ return
def get_best_format(sources: dict[str, str]) -> Format: