Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: simplify some json_ld usage

NTFSvolume1bdd48778bc9
diff --git a/cyberdrop_dl/crawlers/apkmirror.py b/cyberdrop_dl/crawlers/apkmirror.py
index aa0e93ac7..851ecc548 100644
--- a/cyberdrop_dl/crawlers/apkmirror.py
+++ b/cyberdrop_dl/crawlers/apkmirror.py
@@ -65,7 +65,7 @@ class APKMirrorCrawler(Crawler):
def _extract_app_name(soup: BeautifulSoup) -> str:
try:
- _, elem = json_ld.find(soup, validate={"@type": "BreadcrumbList"})
+ elem = json_ld.find_elem(soup, "BreadcrumbList")
return elem["itemListElement"][2]["name"]
except (css.SelectorError, LookupError):
raise css.SelectorError("Unable to extract application name") from None
diff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.py
index 7c253c1e5..ac4a81cfd 100644
--- a/cyberdrop_dl/crawlers/dirtyship.py
+++ b/cyberdrop_dl/crawlers/dirtyship.py
@@ -7,7 +7,7 @@ from cyberdrop_dl import aio
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, parse_url
+from cyberdrop_dl.utils import css, json_ld, parse_url
from cyberdrop_dl.utils.errors import error_handling_wrapper
if TYPE_CHECKING:
@@ -58,8 +58,6 @@ class DirtyShipCrawler(Crawler):
@error_handling_wrapper
async def video(self, scrape_item: ScrapeItem) -> None:
soup = await self.request_soup(scrape_item.url)
- from cyberdrop_dl.utils import json_ld
-
article = json_ld.find_elem(soup, "Article")
title = css.unescape(article["headline"])
scrape_item.uploaded_at = self.parse_iso_date(article["datePublished"])
diff --git a/cyberdrop_dl/crawlers/mydesi.py b/cyberdrop_dl/crawlers/mydesi.py
index f5d3fdc73..79e30234c 100644
--- a/cyberdrop_dl/crawlers/mydesi.py
+++ b/cyberdrop_dl/crawlers/mydesi.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import itertools
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
from cyberdrop_dl.mediaprops import Resolution
@@ -51,7 +51,7 @@ class MyDesiCrawler(Crawler):
resolution, link = max(_parse_formats(soup))
link = self.parse_url(link)
_, ext = self.get_filename_and_ext(link.name)
- _, metadata = json_ld.find(soup, "subjectOf")
+ metadata: dict[str, Any] = json_ld.find_attr(soup, "subjectOf")
title = metadata["name"]
scrape_item.uploaded_at = self.parse_iso_date(metadata["uploadDate"])
diff --git a/cyberdrop_dl/crawlers/yurivan.py b/cyberdrop_dl/crawlers/yurivan.py
index 8f21dbb09..11acec178 100644
--- a/cyberdrop_dl/crawlers/yurivan.py
+++ b/cyberdrop_dl/crawlers/yurivan.py
@@ -42,7 +42,7 @@ class YuriVanCrawler(Crawler):
async def story(self, scrape_item: ScrapeItem, story_id: str) -> None:
soup = await self.request_soup(scrape_item.url)
try:
- _, video_props = json_ld.find(soup, validate={"@type": "VideoObject"})
+ video_props = json_ld.find_elem(soup, "VideoObject")
except css.SelectorError:
scrape_item.setup_as_album("")
self._chapters(scrape_item, story_id, soup)