Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: simplify some json_ld usage
NTFSvolume1bdd48778bc9
diff --git a/cyberdrop_dl/crawlers/apkmirror.py b/cyberdrop_dl/crawlers/apkmirror.pyindex aa0e93ac7..851ecc548 100644--- a/cyberdrop_dl/crawlers/apkmirror.py+++ b/cyberdrop_dl/crawlers/apkmirror.py@@ -65,7 +65,7 @@ class APKMirrorCrawler(Crawler):def _extract_app_name(soup: BeautifulSoup) -> str:try:- _, elem = json_ld.find(soup, validate={"@type": "BreadcrumbList"})+ elem = json_ld.find_elem(soup, "BreadcrumbList")return elem["itemListElement"][2]["name"]except (css.SelectorError, LookupError):raise css.SelectorError("Unable to extract application name") from Nonediff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.pyindex 7c253c1e5..ac4a81cfd 100644--- a/cyberdrop_dl/crawlers/dirtyship.py+++ b/cyberdrop_dl/crawlers/dirtyship.py@@ -7,7 +7,7 @@ from cyberdrop_dl import aiofrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, parse_url+from cyberdrop_dl.utils import css, json_ld, parse_urlfrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:@@ -58,8 +58,6 @@ class DirtyShipCrawler(Crawler):@error_handling_wrapperasync def video(self, scrape_item: ScrapeItem) -> None:soup = await self.request_soup(scrape_item.url)- from cyberdrop_dl.utils import json_ld-article = json_ld.find_elem(soup, "Article")title = css.unescape(article["headline"])scrape_item.uploaded_at = self.parse_iso_date(article["datePublished"])diff --git a/cyberdrop_dl/crawlers/mydesi.py b/cyberdrop_dl/crawlers/mydesi.pyindex f5d3fdc73..79e30234c 100644--- a/cyberdrop_dl/crawlers/mydesi.py+++ b/cyberdrop_dl/crawlers/mydesi.py@@ -1,7 +1,7 @@from __future__ import annotationsimport itertools-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPathsfrom cyberdrop_dl.mediaprops import Resolution@@ -51,7 +51,7 @@ class MyDesiCrawler(Crawler):resolution, link = max(_parse_formats(soup))link = self.parse_url(link)_, ext = self.get_filename_and_ext(link.name)- _, metadata = json_ld.find(soup, "subjectOf")+ metadata: dict[str, Any] = json_ld.find_attr(soup, "subjectOf")title = metadata["name"]scrape_item.uploaded_at = self.parse_iso_date(metadata["uploadDate"])diff --git a/cyberdrop_dl/crawlers/yurivan.py b/cyberdrop_dl/crawlers/yurivan.pyindex 8f21dbb09..11acec178 100644--- a/cyberdrop_dl/crawlers/yurivan.py+++ b/cyberdrop_dl/crawlers/yurivan.py@@ -42,7 +42,7 @@ class YuriVanCrawler(Crawler):async def story(self, scrape_item: ScrapeItem, story_id: str) -> None:soup = await self.request_soup(scrape_item.url)try:- _, video_props = json_ld.find(soup, validate={"@type": "VideoObject"})+ video_props = json_ld.find_elem(soup, "VideoObject")except css.SelectorError:scrape_item.setup_as_album("")self._chapters(scrape_item, story_id, soup)