Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: rework css module (#31)
NTFSvolumec90ebdfc54bc
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex f499a1a56..55db0ff70 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -207,7 +207,7 @@ class CheveretoCrawler(Crawler, is_generic=True):def _get_album_files(self, soup: BeautifulSoup) -> Generator[tuple[AbsoluteHttpURL, AbsoluteHttpURL]]:for item in soup.select(".list-item[data-object]"):web_url = self.parse_url(css.select(item, "a.image-container", "href"))- encoded_data = css.get_attr(item, "data-object")+ encoded_data = css.attr(item, "data-object")data = json.loads(urllib.parse.unquote(encoded_data))src_url = self.parse_url(data["image"]["url"])yield self.transform_url(web_url), src_urldiff --git a/cyberdrop_dl/crawlers/_fluid_player.py b/cyberdrop_dl/crawlers/_fluid_player.pyindex 7089dc4ed..824c34c79 100644--- a/cyberdrop_dl/crawlers/_fluid_player.py+++ b/cyberdrop_dl/crawlers/_fluid_player.py@@ -39,7 +39,7 @@ class FluidPlayerCrawler(Crawler, is_abc=True):link = self.parse_url(best_format.link_str)filename, ext = self.get_filename_and_ext(link.name)title = open_graph.title(soup)- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=best_format.resolution)return await self.handle_file(scrape_item.url, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=link@@ -68,8 +68,8 @@ def _get_best_format(soup: BeautifulSoup) -> Format:def parse():for src in soup.select(Selector.VIDEO_SRC):- url = css.get_attr(src, "src")- quality = css.get_attr_or_none(src, "title")+ url = css.attr(src, "src")+ quality = css.attr_or_none(src, "title")resolution = parse_resolution(quality)yield Format(resolution, url)diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex 9f28cc2a7..cb9b7f6fa 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -88,7 +88,7 @@ class ForumPost:except Exception:date = None- id_str = css.get_attr(article, selectors.id.attribute)+ id_str = css.attr(article, selectors.id.attribute)post_id = int(id_str.rsplit("-", 1)[-1])return ForumPost(post_id, date, article, content)@@ -532,14 +532,17 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):def _lazy_load_embeds(self, post: ForumPostProtocol) -> Iterable[str]:selector = self.SELECTORS.posts.lazy_load_embedsfor lazy_media in css.iselect(post.content, selector.element):- yield get_text_between(css.get_attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")+ yield get_text_between(css.attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:async for soup in self._web_pager(scrape_item.url, self.get_next_page):yield soupdef get_next_page(self, soup: BeautifulSoup) -> str | None:- return css.select_one_get_attr_or_none(soup, *self.SELECTORS.next_page)+ try:+ return css.select(soup, *self.SELECTORS.next_page)+ except css.SelectorError:+ return@final@error_handling_wrapper@@ -579,7 +582,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):if not confirm_button:return- link_str: str = css.get_attr(confirm_button, selector.attribute)+ link_str: str = css.attr(confirm_button, selector.attribute)link_str = link_str.split('" class="link link--internal', 1)[0]new_link = self.parse_url(link_str)return await self.get_absolute_link(new_link)@@ -596,7 +599,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):if link_obj.select_one(".username"):return Truetry:- if link_str := css.get_attr(link_obj, self.SELECTORS.posts.links.element):+ if link_str := css.attr(link_obj, self.SELECTORS.posts.links.element):return self.is_attachment(link_str)except Exception:pass@@ -606,7 +609,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):def iter_links(links: Iterable[Tag], attribute: str) -> Iterable[str]:for link_tag in links:try:- yield css.get_attr(link_tag, attribute)+ yield css.attr(link_tag, attribute)except Exception:continue@@ -674,7 +677,7 @@ def get_post_title(soup: BeautifulSoup, selectors: MessageBoardSelectors) -> strexcept (AttributeError, AssertionError, css.SelectorError) as e:raise ScrapeError(429, message="Invalid response from forum. You may have been rate limited") from e- if title := " ".join(css.get_text(title_block).split()):+ if title := " ".join(css.text(title_block).split()):return titleraise ScrapeError(422)diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex c88442786..812c548aa 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -129,7 +129,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):filename, ext = self.get_filename_and_ext(video.url.name)custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=video.resolution)try:- date_str = css.get_json_ld_date(soup)+ date_str = css.json_ld(soup)["uploadDate"]scrape_item.possible_datetime = self.parse_iso_date(date_str)except (LookupError, ValueError, css.SelectorError):date_str = css.select_text(soup, _SELECTORS.DATE).split(":", 1)[-1].strip()@@ -173,7 +173,7 @@ def extract_kvs_video(cls: Crawler, soup: BeautifulSoup) -> KVSVideo:if not video.title:title = open_graph.get_title(soup) or css.page_title(soup)assert title- video.title = css.sanitize_page_title(title, cls.DOMAIN)+ video.title = css.rstrip_domain(title, cls.DOMAIN)return videodiff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.pyindex b0e84dfd9..c3bb54ed8 100644--- a/cyberdrop_dl/crawlers/_yetishare.py+++ b/cyberdrop_dl/crawlers/_yetishare.py@@ -109,8 +109,8 @@ class YetiShareCrawler(Crawler, is_abc=True):total_pages = int(css.select(ajax_soup, Selector.FOLDER_TOTAL_PAGES, "value"))for file in ajax_soup.select(Selector.FILES):- file_url = self.parse_url(css.get_attr(file, "dtfullurl"))- content_id = int(css.get_attr(file, "fileid"))+ file_url = self.parse_url(css.attr(file, "dtfullurl"))+ content_id = int(css.attr(file, "fileid"))new_scrape_item = scrape_item.create_child(file_url)new_scrape_item.part_of_album = not is_sharedself.create_task(self._handle_content_id_task(new_scrape_item, content_id))@@ -157,7 +157,7 @@ class YetiShareCrawler(Crawler, is_abc=True):# Manually parse link. Some URLs are invalid. ex: https://cyberfile.me/7cfu# For the download URL, the slug does not actually matter. It can be anything- raw_link = get_text_between(css.get_attr(download_tag, "onclick"), "('", "');")+ raw_link = get_text_between(css.attr(download_tag, "onclick"), "('", "');")token = raw_link.rpartition("?download_token=")[-1]link = self.parse_url(raw_link).with_query(download_token=token)diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.pyindex 46aa019c9..7136ff311 100644--- a/cyberdrop_dl/crawlers/ashemaletube.py+++ b/cyberdrop_dl/crawlers/ashemaletube.py@@ -154,13 +154,13 @@ class AShemaleTubeCrawler(Crawler):@error_handling_wrapperasync def proccess_image(self, scrape_item: ScrapeItem, img_tag: Tag) -> None:if image := img_tag.select_one("img"):- link_str: str = css.get_attr(image, "src")+ link_str: str = css.attr(image, "src")else:style: str = css.select(img_tag, "a", "style")link_str = get_text_between(style, "url('", "');")url = self.parse_url(link_str).with_query(None)filename, ext = self.get_filename_and_ext(url.name)- custom_filename = self.create_custom_filename(filename, ext, file_id=css.get_attr(img_tag, "data-image-id"))+ custom_filename = self.create_custom_filename(filename, ext, file_id=css.attr(img_tag, "data-image-id"))await self.handle_file(url, scrape_item, filename, ext, custom_filename=custom_filename)@error_handling_wrapper@@ -183,7 +183,7 @@ class AShemaleTubeCrawler(Crawler):debrid_link = best_format.urlif video_object := soup.select_one(_SELECTORS.VIDEO_PROPS_JS):- json_data = json.loads(css.get_text(video_object))+ json_data = json.loads(css.text(video_object))scrape_item.possible_datetime = self.parse_iso_date(json_data.get("uploadDate", ""))title = css.select_text(soup, "title").split("- aShemaletube.com")[0].strip()diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.pyindex bc7d617cd..603199ba0 100644--- a/cyberdrop_dl/crawlers/bunkrr.py+++ b/cyberdrop_dl/crawlers/bunkrr.py@@ -205,7 +205,7 @@ class BunkrrCrawler(Crawler):soup = await self._request_soup_lenient(scrape_item.url)src = Noneif image := soup.select_one(Selector.IMAGE_PREVIEW):- src = self.parse_url(css.get_attr(image, "src"))+ src = self.parse_url(css.attr(image, "src"))if len(src.parts) > 2:src = Nonediff --git a/cyberdrop_dl/crawlers/camwhores_dot_tv.py b/cyberdrop_dl/crawlers/camwhores_dot_tv.pyindex 43a87b654..24ccf254a 100644--- a/cyberdrop_dl/crawlers/camwhores_dot_tv.py+++ b/cyberdrop_dl/crawlers/camwhores_dot_tv.py@@ -53,7 +53,7 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler):soup = await self.request_soup(url)try:- last_page = int(css.get_text(soup.select(LAST_PAGE_SELECTOR)[-1]))+ last_page = int(css.text(soup.select(LAST_PAGE_SELECTOR)[-1]))except IndexError:returndiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 9a62316c0..6ba0dc823 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -8,7 +8,7 @@ import refrom abc import ABC, abstractmethodfrom collections import Counterfrom dataclasses import dataclass, field-from functools import partial, wraps+from functools import wrapsfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Literal, NamedTuple, ParamSpec, TypeAlias, TypeVar, final@@ -652,7 +652,7 @@ class Crawler(ABC):seen: set[str] = set()for tag in css.iselect(soup, selector):- link_str: str | None = css.get_attr_or_none(tag, attribute)+ link_str: str | None = css.attr_or_none(tag, attribute)if not link_str or link_str in seen:continueseen.add(link_str)@@ -660,7 +660,7 @@ class Crawler(ABC):if self.check_album_results(link, album_results):continueif t_tag := tag.select_one("img"):- thumb_str: str | None = css.get_attr_or_none(t_tag, "src")+ thumb_str: str | None = css.attr_or_none(t_tag, "src")else:thumb_str = Nonethumb = self.parse_url(thumb_str) if thumb_str and not is_blob_or_svg(thumb_str) else None@@ -746,8 +746,12 @@ class Crawler(ABC):else:selector = selector or self.NEXT_PAGE_SELECTORassert selector, f"No selector was provided and {self.DOMAIN} does define a next_page_selector"- func = css.select_one_get_attr_or_none- get_next_page = partial(func, selector=selector, attribute="href")++ def get_next_page(soup: BeautifulSoup, /) -> str | None:+ try:+ return css.select(soup, selector, "href")+ except css.SelectorError:+ returnwhile True:soup = await self.request_soup(page_url, impersonate=cffi or None)diff --git a/cyberdrop_dl/crawlers/desivideo.py b/cyberdrop_dl/crawlers/desivideo.pyindex e9d6ad4f0..a37611992 100644--- a/cyberdrop_dl/crawlers/desivideo.py+++ b/cyberdrop_dl/crawlers/desivideo.py@@ -46,7 +46,7 @@ class DesiVideoCrawler(Crawler):video_url = self.parse_url(Selector.VIDEO_SRC(soup))title = css.select_text(soup, Selector.TITLE)_, ext = self.get_filename_and_ext(video_url.name)- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])custom_filename = self.create_custom_filename(title, ext, file_id=video_id)return await self.handle_file(video_url, scrape_item, video_url.name, ext, custom_filename=custom_filename)diff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.pyindex 52093e02b..4bddcb89f 100644--- a/cyberdrop_dl/crawlers/dirtyship.py+++ b/cyberdrop_dl/crawlers/dirtyship.py@@ -68,7 +68,7 @@ class DirtyShipCrawler(Crawler):if not scrape_item.url.suffix == ".jpg":soup = await self.request_soup(scrape_item.url)url = self.parse_url(- next(css.get_attr(a, "href") for a in soup.select(_SELECTORS.SINGLE_PHOTO) if "full" in a.get_text())+ next(css.attr(a, "href") for a in soup.select(_SELECTORS.SINGLE_PHOTO) if "full" in a.get_text()))else:url = scrape_item.url@@ -96,9 +96,9 @@ class DirtyShipCrawler(Crawler):for img in thumbnails:url = (- css.get_attr(img, "src")+ css.attr(img, "src")if img.get("decoding") == "async"- else get_highest_resolution_picture(css.get_attr(img, "srcset"))+ else get_highest_resolution_picture(css.attr(img, "srcset")))if not url:raise ScrapeError(404)@@ -129,10 +129,10 @@ class DirtyShipCrawler(Crawler):def get_formats():for video in videos:- link_str: str = css.get_attr(video, "src")+ link_str: str = css.attr(video, "src")if link_str.startswith("type="):continue- res: str = css.get_attr(video, "title")+ res: str = css.attr(video, "title")link = self.parse_url(link_str)yield (Format(int(res), link))@@ -149,7 +149,7 @@ class DirtyShipCrawler(Crawler):def get_flowplayer_sources(self, soup: BeautifulSoup) -> set[Format]:flow_player = soup.select_one(_SELECTORS.FLOWPLAYER_VIDEO)- data_item: str | None = css.get_attr_or_none(flow_player, "data-item") if flow_player else None+ data_item: str | None = css.attr_or_none(flow_player, "data-item") if flow_player else Noneif not data_item:return set()data_item = data_item.replace(r"\/", "/")diff --git a/cyberdrop_dl/crawlers/eightmuses.py b/cyberdrop_dl/crawlers/eightmuses.pyindex 4fd2ae7fc..cf686e9d1 100644--- a/cyberdrop_dl/crawlers/eightmuses.py+++ b/cyberdrop_dl/crawlers/eightmuses.py@@ -46,8 +46,8 @@ class EightMusesCrawler(Crawler):results = await self.get_album_results(album_id)for tile in soup.select(TILE_SELECTOR):- tile_link = self.parse_url(css.get_attr(tile, "href"))- tile_title: str = css.get_attr_or_none(tile, "title") or ""+ tile_link = self.parse_url(css.attr(tile, "href"))+ tile_title: str = css.attr_or_none(tile, "title") or ""image = css.select(tile, IMAGE_SELECTOR)is_new_album = image["itemtype"] == "https://schema.org/ImageGallery"new_album_id = f"{scrape_item.album_id}/{tile_title.replace(' ', '-')}"diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.pyindex 380fde314..260ce34ad 100644--- a/cyberdrop_dl/crawlers/eporner.py+++ b/cyberdrop_dl/crawlers/eporner.py@@ -62,7 +62,7 @@ class VideoSource:@staticmethoddef parse(tag: Tag) -> VideoSource:- link_str: str = css.get_attr(tag, "href")+ link_str: str = css.attr(tag, "href")name = tag.get_text(strip=True).removeprefix("Download")details = name.split("(", 1)[1].removesuffix(")").split(",")res, codec, size = [d.strip() for d in details]diff --git a/cyberdrop_dl/crawlers/fapello.py b/cyberdrop_dl/crawlers/fapello.pyindex f77463ca4..0f44fed5c 100644--- a/cyberdrop_dl/crawlers/fapello.py+++ b/cyberdrop_dl/crawlers/fapello.py@@ -45,7 +45,7 @@ class FapelloCrawler(Crawler):scrape_item.setup_as_album(title)for post in soup.select(CONTENT_SELECTOR):- link_str: str = css.get_attr(post, "href")+ link_str: str = css.attr(post, "href")if "javascript" in link_str:link_str = css.select(post, "iframe", "src")diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.pyindex 774dcf40d..0bfc4daa7 100644--- a/cyberdrop_dl/crawlers/filester.py+++ b/cyberdrop_dl/crawlers/filester.py@@ -54,7 +54,7 @@ class FilesterCrawler(Crawler):while True:for row in soup.select(Selector.FILES):- url = get_text_between(css.get_attr(row, "onclick"), "'", "'")+ url = get_text_between(css.attr(row, "onclick"), "'", "'")web_url = self.parse_url(url)new_scrape_item = scrape_item.create_child(web_url)self.create_task(self.run(new_scrape_item))diff --git a/cyberdrop_dl/crawlers/fsiblog.py b/cyberdrop_dl/crawlers/fsiblog.pyindex 9f4e6153d..66f511a3a 100644--- a/cyberdrop_dl/crawlers/fsiblog.py+++ b/cyberdrop_dl/crawlers/fsiblog.py@@ -55,7 +55,7 @@ class FSIBlogCrawler(Crawler):@error_handling_wrapperasync def post(self, scrape_item: ScrapeItem) -> None:soup = await self.request_soup(scrape_item.url)- meta: dict[str, str] = css.get_json_ld(soup)["@graph"][0]+ meta: dict[str, str] = css.json_ld(soup)["@graph"][0]name = meta["name"].rpartition("-")[0]scrape_item.possible_datetime = date = self.parse_iso_date(meta["datePublished"])title = self.create_separate_post_title(name, None, date)diff --git a/cyberdrop_dl/crawlers/hianime.py b/cyberdrop_dl/crawlers/hianime.pyindex 8d103b197..43969c3c0 100644--- a/cyberdrop_dl/crawlers/hianime.py+++ b/cyberdrop_dl/crawlers/hianime.py@@ -40,10 +40,10 @@ class Episode:@staticmethoddef from_tag(ep_tag: bs4.Tag) -> Episode:return Episode(- title=css.get_attr(ep_tag, "title"),- number=int(css.get_attr(ep_tag, "data-number")),- id=int(css.get_attr(ep_tag, "data-id")),- path_qs=css.get_attr(ep_tag, "href"),+ title=css.attr(ep_tag, "title"),+ number=int(css.attr(ep_tag, "data-number")),+ id=int(css.attr(ep_tag, "data-id")),+ path_qs=css.attr(ep_tag, "href"),)@@ -169,5 +169,5 @@ def _parse_server_resp(html: str):soup = bs4.BeautifulSoup(html, "html.parser")for server_type in ("sub", "dub", "raw"):if server_tag := soup.select_one(f"div[data-type={server_type}]:-soup-contains('HD-1')"):- server_id = css.get_attr(server_tag, "data-id")+ server_id = css.attr(server_tag, "data-id")yield server_type, server_iddiff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.pyindex 6ff81c5de..8d92d9712 100644--- a/cyberdrop_dl/crawlers/imagebam.py+++ b/cyberdrop_dl/crawlers/imagebam.py@@ -85,8 +85,9 @@ class ImageBamCrawler(Crawler):for _, new_scrape_item in self.iter_children(scrape_item, soup, Selectors.THUMBNAILS, results=results):self.create_task(self._image_task(new_scrape_item))- next_page = css.select_one_get_attr_or_none(soup, Selectors.NEXT_PAGE, "href")- if not next_page:+ try:+ next_page = css.select(soup, Selectors.NEXT_PAGE, "href")+ except css.SelectorError:breaksoup = await self.request_soup(self.parse_url(next_page))@@ -100,11 +101,11 @@ class ImageBamCrawler(Crawler):image_tag = css.select(soup, Selectors.IMAGE)if not scrape_item.album_id and (gallery_info := soup.select_one(Selectors.GALLERY_INFO)):- gallery_id = self.parse_url(css.get_attr(gallery_info, "href")).name+ gallery_id = self.parse_url(css.attr(gallery_info, "href")).namescrape_item.album_id = gallery_id- title = css.get_attr(image_tag, "alt")- link = self.parse_url(css.get_attr(image_tag, "src"))+ title = css.attr(image_tag, "alt")+ link = self.parse_url(css.attr(image_tag, "src"))custom_filename, ext = self.get_filename_and_ext(title)await self.handle_file(link, scrape_item, link.name, ext, custom_filename=custom_filename)diff --git a/cyberdrop_dl/crawlers/imgbox.py b/cyberdrop_dl/crawlers/imgbox.pyindex e0a2f8088..6df62f116 100644--- a/cyberdrop_dl/crawlers/imgbox.py+++ b/cyberdrop_dl/crawlers/imgbox.py@@ -47,7 +47,7 @@ class ImgBoxCrawler(Crawler):scrape_item.setup_as_album(title, album_id=album_id)for link in soup.select(IMAGES_SELECTOR):- link_str: str = css.get_attr(link, "src").replace("thumbs", "images").replace("_b", "_o")+ link_str: str = css.attr(link, "src").replace("thumbs", "images").replace("_b", "_o")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.pyindex 895bc481a..7f30982d7 100644--- a/cyberdrop_dl/crawlers/imx_to.py+++ b/cyberdrop_dl/crawlers/imx_to.py@@ -64,8 +64,8 @@ class ImxToCrawler(Crawler):data={"imgContinue": "Continue+to+image+...+"},)image = css.select(soup, Selector.IMAGES)- name = css.get_attr(image, "alt")- link = self.parse_url(css.get_attr(image, "src"))+ name = css.attr(image, "alt")+ link = self.parse_url(css.attr(image, "src"))filename, ext = self.get_filename_and_ext(name)await self.handle_file(link, scrape_item, name, ext, custom_filename=filename)diff --git a/cyberdrop_dl/crawlers/influencer_bitches.py b/cyberdrop_dl/crawlers/influencer_bitches.pyindex 55e4a1dc0..e424b596d 100644--- a/cyberdrop_dl/crawlers/influencer_bitches.py+++ b/cyberdrop_dl/crawlers/influencer_bitches.py@@ -58,7 +58,7 @@ class InfluencerBitchesCrawler(Crawler):link = self.parse_url(link_str)if self.check_album_results(link, results):continue- web_url = self.parse_url(css.get_attr(a_tag, "href"))+ web_url = self.parse_url(css.attr(a_tag, "href"))new_scrape_item = scrape_item.create_child(web_url)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, new_scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.pyindex 10a11a5b3..22961e87b 100644--- a/cyberdrop_dl/crawlers/kemono.py+++ b/cyberdrop_dl/crawlers/kemono.py@@ -578,7 +578,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):PostSelectors.ATTACHMENTS,):for file in soup.select(selector):- yield self.parse_url(css.get_attr(file, "href"))+ yield self.parse_url(css.attr(file, "href"))post = UserPost(user_id=user_id,diff --git a/cyberdrop_dl/crawlers/luxuretv.py b/cyberdrop_dl/crawlers/luxuretv.pyindex 1c4007266..55cdfdac9 100644--- a/cyberdrop_dl/crawlers/luxuretv.py+++ b/cyberdrop_dl/crawlers/luxuretv.py@@ -62,10 +62,10 @@ class LuxureTVCrawler(Crawler):returnsoup = await self.request_soup(scrape_item.url, impersonate=True)- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])video_player = css.select(soup, Selector.VIDEO_PLAYER)title = css.select_text(soup, Selector.TITLE)- link = self.parse_url(css.get_attr(video_player, "src"))+ link = self.parse_url(css.attr(video_player, "src"))filename, ext = self.get_filename_and_ext(link.name)custom_filename = self.create_custom_filename(title, ext, file_id=video_id)diff --git a/cyberdrop_dl/crawlers/masahub.py b/cyberdrop_dl/crawlers/masahub.pyindex e2dca8eb1..9dda4c47f 100644--- a/cyberdrop_dl/crawlers/masahub.py+++ b/cyberdrop_dl/crawlers/masahub.py@@ -45,7 +45,7 @@ class MasahubCrawler(Crawler):link = self.parse_url(Selector.VIDEO_SRC(soup))title = css.select_text(soup, Selector.TITLE)_, ext = self.get_filename_and_ext(link.name)- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])custom_filename = self.create_custom_filename(title, ext)return await self.handle_file(link, scrape_item, link.name, ext, custom_filename=custom_filename)diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 2565afbe3..0e580d00b 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -214,10 +214,10 @@ def _extract_download_link(soup: BeautifulSoup) -> str:raise ScrapeError(410)raise ScrapeError(422)- if encoded_url := css.get_attr_or_none(download_button, "data-scrambled-url"):+ if encoded_url := css.attr_or_none(download_button, "data-scrambled-url"):return base64.b64decode(encoded_url).decode()- url = css.get_attr(download_button, "href")+ url = css.attr(download_button, "href")if is_blob_or_svg(url):raise ScrapeError(422)return urldiff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.pyindex 2c4425eda..0f86271a0 100644--- a/cyberdrop_dl/crawlers/missav.py+++ b/cyberdrop_dl/crawlers/missav.py@@ -69,7 +69,7 @@ class MissAVCrawler(Crawler):if date_str:scrape_item.possible_datetime = self.parse_iso_date(date_str)elif date_tag := soup.select_one(Selector.DATE):- scrape_item.possible_datetime = self.parse_iso_date(css.get_attr(date_tag, "datetime"))+ scrape_item.possible_datetime = self.parse_iso_date(css.attr(date_tag, "datetime"))else:_ = self.parse_date("") # Trigger warning@@ -89,7 +89,7 @@ def _get_uuid(soup: BeautifulSoup) -> str:def _fix_title(title: str, dvd_code_tag: Tag) -> str:- dvd_code = css.get_text(dvd_code_tag).upper()+ dvd_code = css.text(dvd_code_tag).upper()uncensored = "UNCENSORED" in dvd_codeleak = "LEAK" in dvd_codefor trash in ("-UNCENSORED", "-LEAK"):diff --git a/cyberdrop_dl/crawlers/motherless.py b/cyberdrop_dl/crawlers/motherless.pyindex b554fc0a3..639120366 100644--- a/cyberdrop_dl/crawlers/motherless.py+++ b/cyberdrop_dl/crawlers/motherless.py@@ -172,7 +172,7 @@ class MotherlessCrawler(Crawler):title_tag = soup.select_one(GROUP_TITLE_SELECTOR)if title_tag:- parent_title: str = css.get_attr(title_tag, "title") if from_gallery else title_tag.get_text(strip=True)+ parent_title: str = css.attr(title_tag, "title") if from_gallery else title_tag.get_text(strip=True)parent_path = parent_id if from_gallery else f"g/{parent_id}"parent_url = PRIMARY_URL / parent_path@@ -195,7 +195,7 @@ def check_soup(soup: BeautifulSoup) -> None:def get_media_info(soup: BeautifulSoup) -> MediaInfo:media_js = soup.select_one(MEDIA_INFO_JS_SELECTOR)- js_text = css.get_text(media_js) if media_js else None+ js_text = css.text(media_js) if media_js else Noneif not js_text:return MediaInfo("gallery", "")media_type = js_text.split("__mediatype", 1)[-1].split("=", 1)[-1].split(",", 1)[0].strip()diff --git a/cyberdrop_dl/crawlers/mydesi.py b/cyberdrop_dl/crawlers/mydesi.pyindex cb47f8f22..3bddd98de 100644--- a/cyberdrop_dl/crawlers/mydesi.py+++ b/cyberdrop_dl/crawlers/mydesi.py@@ -51,7 +51,7 @@ class MyDesiCrawler(Crawler):resolution, link = max(_parse_formats(soup))link = self.parse_url(link)_, ext = self.get_filename_and_ext(link.name)- metadata: dict[str, str] = css.get_json_ld(soup)["subjectOf"]+ metadata: dict[str, str] = css.json_ld(soup)["subjectOf"]title = metadata["name"]scrape_item.possible_datetime = self.parse_iso_date(metadata.get("uploadDate", ""))custom_filename = self.create_custom_filename(title, ext, resolution=resolution)@@ -76,7 +76,7 @@ class MyDesiCrawler(Crawler):def _parse_formats(soup: BeautifulSoup) -> Generator[tuple[Resolution, str]]:for src in soup.select("#video-rate > a"):- quality = css.get_attr(src, "title")- link = css.get_attr(src, "href")+ quality = css.attr(src, "title")+ link = css.attr(src, "href")resolution = Resolution.highest() if "original" in quality.casefold() else Resolution.parse(quality)yield resolution, linkdiff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.pyindex de0c7eddb..e2d5f1800 100644--- a/cyberdrop_dl/crawlers/odnoklassniki.py+++ b/cyberdrop_dl/crawlers/odnoklassniki.py@@ -76,7 +76,12 @@ class OdnoklassnikiCrawler(Crawler):channel_id = channel_str.removeprefix("c")gwt_hash = get_text_between(css.select_text(soup, Selector.CHANNEL_HASH), 'gwtHash:"', '",')- last_element_id = css.select_one_get_attr_or_none(soup, *Selector.CHANNEL_LAST_ELEMENT)++ try:+ last_element_id = css.select(soup, *Selector.CHANNEL_LAST_ELEMENT)+ except css.SelectorError:+ last_element_id = None+name = css.select_text(soup, Selector.CHANNEL_NAME)scrape_item.setup_as_album(self.create_title(name, channel_id), album_id=channel_id)@@ -139,7 +144,7 @@ class OdnoklassnikiCrawler(Crawler):cdn_url = self.parse_url(src)# downloads may fail if we have cdn cookiesself.client.client_manager.cookies.clear_domain(cdn_url.host)- json_ld = css.get_json_ld(soup)+ json_ld = css.json_ld(soup)title: str = metadata["movie"].get("title") or json_ld["name"]scrape_item.possible_datetime = self.parse_iso_date(json_ld["uploadDate"])filename = self.create_custom_filename(title, ".mp4", file_id=video_id, resolution=resolution)diff --git a/cyberdrop_dl/crawlers/pimp_bunny.py b/cyberdrop_dl/crawlers/pimp_bunny.pyindex 26cbafdf3..9d98fe505 100644--- a/cyberdrop_dl/crawlers/pimp_bunny.py+++ b/cyberdrop_dl/crawlers/pimp_bunny.py@@ -128,7 +128,7 @@ class PimpBunnyCrawler(Crawler):custom_filename = self.create_custom_filename(video.title, video.url.suffix, file_id=video.id, resolution=video.resolution)- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])await self.handle_file(scrape_item.url,scrape_item,diff --git a/cyberdrop_dl/crawlers/pimpandhost.py b/cyberdrop_dl/crawlers/pimpandhost.pyindex 8d824c924..4c40782a8 100644--- a/cyberdrop_dl/crawlers/pimpandhost.py+++ b/cyberdrop_dl/crawlers/pimpandhost.py@@ -42,7 +42,7 @@ class PimpAndHostCrawler(Crawler):scrape_item.setup_as_album(title, album_id=album_id)if date_tag := soup.select_one(DATE_SELECTOR):- scrape_item.possible_datetime = self.parse_date(css.get_attr(date_tag, "title"), DATE_FORMAT)+ scrape_item.possible_datetime = self.parse_date(css.attr(date_tag, "title"), DATE_FORMAT)for _, new_scrape_item in self.iter_children(scrape_item, soup, FILES_SELECTOR):self.create_task(self.run(new_scrape_item))diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.pyindex bdbd161fa..f581332af 100644--- a/cyberdrop_dl/crawlers/pkmncards.py+++ b/cyberdrop_dl/crawlers/pkmncards.py@@ -138,8 +138,8 @@ class PkmncardsCrawler(Crawler):async for soup in self.web_pager(page_url):for thumb in soup.select(_SELECTORS.CARD):link_str = css.select(thumb, "img", "src")- card_page_url_str = css.get_attr(thumb, "href")- title = css.get_attr(thumb, "title")+ card_page_url_str = css.attr(thumb, "href")+ title = css.attr(thumb, "title")card_page_url = self.parse_url(card_page_url_str)download_url = self.parse_url(link_str)simple_card = create_simple_card(title, download_url)diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.pyindex 3e96b3269..121961078 100644--- a/cyberdrop_dl/crawlers/pmvhaven.py+++ b/cyberdrop_dl/crawlers/pmvhaven.py@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css+from cyberdrop_dl.utils import css, nuxtfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -53,14 +53,14 @@ class PMVHavenCrawler(Crawler):title = self.create_title(title)scrape_item.setup_as_profile(title)- nuxt_data = css.get_nuxt_data(soup)+ nuxt_data = nuxt.extract(soup)await self._process_video_list(scrape_item, nuxt_data)@error_handling_wrapperasync def playlist(self, scrape_item: ScrapeItem) -> None:soup = await self.request_soup(scrape_item.url)- nuxt_data = css.get_nuxt_data(soup)- playlist = css.parse_nuxt_obj(nuxt_data, "playlist")+ nuxt_data = nuxt.extract(soup)+ playlist = nuxt.parse_obj(nuxt_data, "playlist")name = playlist["name"]title = self.create_title(f"{name} [playlist]")scrape_item.setup_as_album(title)@@ -72,7 +72,7 @@ class PMVHavenCrawler(Crawler):tags = scrape_item.url.query.get("tags") or scrape_item.url.query.get("musicSong")title = self.create_title(f"{tags} [search]")scrape_item.setup_as_album(title)- nuxt_data = css.get_nuxt_data(soup)+ nuxt_data = nuxt.extract(soup)await self._process_video_list(scrape_item, nuxt_data)@error_handling_wrapper@@ -81,15 +81,15 @@ class PMVHavenCrawler(Crawler):returnsoup = await self.request_soup(scrape_item.url)- nuxt_data = css.get_nuxt_data(soup)- video = css.parse_nuxt_obj(nuxt_data, "video", "uploaderVideosCount")+ nuxt_data = nuxt.extract(soup)+ video = nuxt.parse_obj(nuxt_data, "video", "uploaderVideosCount")await self._process_video_info(scrape_item, video)# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~@error_handling_wrapperasync def _process_video_list(self, scrape_item: ScrapeItem, nuxt_data: list[Any]) -> None:- for video_info in css.parse_nuxt_objs(nuxt_data, "videoUrl"):+ for video_info in nuxt.parse_objs(nuxt_data, "videoUrl"):await self._process_video_info(scrape_item, video_info)@error_handling_wrapperdiff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.pyindex bcafd761c..596bd0e75 100644--- a/cyberdrop_dl/crawlers/pornhub.py+++ b/cyberdrop_dl/crawlers/pornhub.py@@ -205,8 +205,8 @@ class PornHubCrawler(Crawler):link_str: str = css.select(soup, _SELECTORS.PHOTO, "src")link = self.parse_url(link_str)album_tag = css.select(soup, _SELECTORS.ALBUM_FROM_PHOTO)- album_name = css.get_text(album_tag)- album_link_str: str = css.get_attr(album_tag, "href")+ album_name = css.text(album_tag)+ album_link_str: str = css.attr(album_tag, "href")album_id: str = album_link_str.split("/")[-1]title = self.create_title(album_name, album_id)scrape_item.setup_as_album(title, album_id=album_id)@@ -217,7 +217,7 @@ class PornHubCrawler(Crawler):soup = await self.request_soup(scrape_item.url)attributes = "data-mp4", "data-fallback", "data-webm"gif_tag = css.select(soup, _SELECTORS.GIF)- link_str = next(value for attr in attributes if (value := css.get_attr_or_none(gif_tag, attr)))+ link_str = next(value for attr in attributes if (value := css.attr_or_none(gif_tag, attr)))link = self.parse_url(link_str)await self._process_photo(scrape_item, link)diff --git a/cyberdrop_dl/crawlers/pornpics.py b/cyberdrop_dl/crawlers/pornpics.pyindex d6450f0d1..b3481b0b2 100644--- a/cyberdrop_dl/crawlers/pornpics.py+++ b/cyberdrop_dl/crawlers/pornpics.py@@ -108,7 +108,7 @@ class PornPicsCrawler(Crawler):if not offset: # offset == 0 does not return JSONsoup = await self.request_soup(current_page)items = soup.select(IMAGE_SELECTOR)- return soup, tuple(self.parse_url(css.get_attr(image, "href")) for image in items)+ return soup, tuple(self.parse_url(css.attr(image, "href")) for image in items)# The response is JSON but the "content-type" is wrongasync with self.request(current_page) as resp:diff --git a/cyberdrop_dl/crawlers/pornzog.py b/cyberdrop_dl/crawlers/pornzog.pyindex c324eaa67..e4553dfe4 100644--- a/cyberdrop_dl/crawlers/pornzog.py+++ b/cyberdrop_dl/crawlers/pornzog.py@@ -37,6 +37,6 @@ class PornZogCrawler(Crawler):async def video(self, scrape_item: ScrapeItem) -> None:soup = await self.request_soup(scrape_item.url)iframe = css.select(soup, Selectors.EMBED_IFRAME)- external_url = self.parse_url(css.get_attr(iframe, "src"))+ external_url = self.parse_url(css.attr(iframe, "src"))new_scrape_item = scrape_item.create_child(external_url)return self.handle_external_links(new_scrape_item)diff --git a/cyberdrop_dl/crawlers/rule34vault.py b/cyberdrop_dl/crawlers/rule34vault.pyindex 87049a653..8515cc4b3 100644--- a/cyberdrop_dl/crawlers/rule34vault.py+++ b/cyberdrop_dl/crawlers/rule34vault.py@@ -83,7 +83,7 @@ class Rule34VaultCrawler(Crawler):scrape_item.url = canonical_urlmedia_tag = soup.select_one(_SELECTORS.VIDEO) or soup.select_one(_SELECTORS.IMAGE)assert media_tag- link_str: str = css.get_attr(media_tag, "src")+ link_str: str = css.attr(media_tag, "src")for trash in (".small", ".thumbnail", ".picsmall", ".720", ".hevc"):link_str = link_str.replace(trash, "")diff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.pyindex 28f899392..8863deeb7 100644--- a/cyberdrop_dl/crawlers/rule34video.py+++ b/cyberdrop_dl/crawlers/rule34video.py@@ -57,7 +57,7 @@ class Rule34VideoCrawler(KernelVideoSharingCrawler):if not title:title_tag = css.select(soup, Selector.TITLE)css.decompose(title_tag, "span")- title = css.get_text(title_tag)+ title = css.text(title_tag)for trash in ("Videos for: ", "Tagged with "):title = title.removeprefix(trash)diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.pyindex 2b8fee7ea..4e22b72c9 100644--- a/cyberdrop_dl/crawlers/rumble.py+++ b/cyberdrop_dl/crawlers/rumble.py@@ -148,7 +148,7 @@ class RumbleCrawler(Crawler):returnsoup = await self.request_soup(scrape_item.url)- embed_id = self.parse_url(css.get_json_ld(soup)["embedUrl"]).name+ embed_id = self.parse_url(css.json_ld(soup)["embedUrl"]).nameawait self.embed(scrape_item, embed_id)@error_handling_wrapper@@ -161,7 +161,7 @@ class RumbleCrawler(Crawler):_, ext = self.get_filename_and_ext(best_format.url.name)video_name = self.create_custom_filename(video.title, ext, file_id=embed_id, resolution=best_format.resolution)- scrape_item.possible_datetime = self.parse_iso_date(video.upload_date)+ scrape_item.possible_datetime = self.parse_iso_date(video["uploadDate"])scrape_item.url = video.urlself.create_task(self.handle_file(diff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.pyindex c85a15d79..cc964ae43 100644--- a/cyberdrop_dl/crawlers/spankbang.py+++ b/cyberdrop_dl/crawlers/spankbang.py@@ -173,7 +173,7 @@ def _parse_video(soup: BeautifulSoup, display_id: str) -> Video:url=url,stream_id=get_text_between(stream_js_text, "ana_video_id = ", ";").strip("'"),stream_key=css.select(soup, "[data-streamkey]", "data-streamkey"),- title=css.get_attr_or_none(title_tag, "title") or css.get_text(title_tag),+ title=css.attr_or_none(title_tag, "title") or css.text(title_tag),)diff --git a/cyberdrop_dl/crawlers/tnaflix.py b/cyberdrop_dl/crawlers/tnaflix.pyindex 344738835..90f960723 100644--- a/cyberdrop_dl/crawlers/tnaflix.py+++ b/cyberdrop_dl/crawlers/tnaflix.py@@ -63,7 +63,7 @@ class TNAFlixCrawler(Crawler):link = self.parse_url(best_format.link_str)filename, ext = self.get_filename_and_ext(link.name)title = open_graph.title(soup)- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=best_format.resolution)return await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)@@ -83,8 +83,8 @@ class TNAFlixCrawler(Crawler):def _get_best_format(soup: BeautifulSoup) -> Format:def parse():for src in soup.select(Selector.VIDEO_SRC):- url = css.get_attr(src, "src")- resolution = Resolution.parse(css.get_attr(src, "size"))+ url = css.attr(src, "src")+ resolution = Resolution.parse(css.attr(src, "size"))yield Format(resolution, url)return max(parse())diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.pyindex e82d55a9d..f4b6ac1bb 100644--- a/cyberdrop_dl/crawlers/tokyomotion.py+++ b/cyberdrop_dl/crawlers/tokyomotion.py@@ -89,7 +89,7 @@ class TokioMotionCrawler(Crawler):raise ScrapeError(422, "Couldn't find video source")scrape_item.possible_datetime = self.parse_date(css.select_text(soup, _SELECTORS.VIDEO_DATE))- link_str = css.get_attr(src, "src")+ link_str = css.attr(src, "src")link = self.parse_url(link_str)title = css.select_text(soup, "title").rsplit(" - TOKYO Motion")[0].strip()filename, ext = f"{video_id}.mp4", ".mp4"@@ -109,7 +109,7 @@ class TokioMotionCrawler(Crawler):raise ScrapeError(401, "Private Photo")raise ScrapeError(422, "Couldn't find image source")- link_str: str = css.get_attr(img, "src")+ link_str: str = css.attr(img, "src")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/toonily.py b/cyberdrop_dl/crawlers/toonily.pyindex 93142500d..a35800aa7 100644--- a/cyberdrop_dl/crawlers/toonily.py+++ b/cyberdrop_dl/crawlers/toonily.py@@ -41,7 +41,7 @@ class ToonilyCrawler(Crawler):title_tag = css.select(soup, Selector.SERIES_TITLE)css.decompose(title_tag, "*")- series_title = self.create_title(css.get_text(title_tag))+ series_title = self.create_title(css.text(title_tag))scrape_item.setup_as_profile(series_title)for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.CHAPTER):self.create_task(self.run(new_scrape_item))@@ -56,7 +56,7 @@ class ToonilyCrawler(Crawler):scrape_item.add_to_parent_title(series_title)scrape_item.setup_as_album(chapter_title)- iso_date = css.get_json_ld(soup)["@graph"][0]["datePublished"]+ iso_date = css.json_ld(soup)["@graph"][0]["datePublished"]scrape_item.possible_datetime = self.parse_iso_date(iso_date)for _, link in self.iter_tags(soup, Selector.IMAGE, "src"):diff --git a/cyberdrop_dl/crawlers/transflix.py b/cyberdrop_dl/crawlers/transflix.pyindex 147d54bde..1919b4e9b 100644--- a/cyberdrop_dl/crawlers/transflix.py+++ b/cyberdrop_dl/crawlers/transflix.py@@ -49,7 +49,7 @@ class TransflixCrawler(Crawler):soup = await self.request_soup(scrape_item.url)title = open_graph.title(soup)video = css.select(soup, _SELECTORS.VIDEO)- link = self.parse_url(css.get_attr(video, "src"))+ link = self.parse_url(css.attr(video, "src"))filename, ext = self.get_filename_and_ext(link.name)scrape_item.possible_datetime = _timestamp_from_filename(link.name)custom_filename = self.create_custom_filename(title, ext, file_id=video_id)diff --git a/cyberdrop_dl/crawlers/turbovid.py b/cyberdrop_dl/crawlers/turbovid.pyindex 46c054c50..1e701b926 100644--- a/cyberdrop_dl/crawlers/turbovid.py+++ b/cyberdrop_dl/crawlers/turbovid.py@@ -61,7 +61,7 @@ class TurboVidCrawler(Crawler):scrape_item.setup_as_album(title, album_id=album_id)for row in soup.select(Selector.ALBUM_FILES):- file_id = css.get_attr(row, "data-id")+ file_id = css.attr(row, "data-id")web_url = self.PRIMARY_URL / "d" / file_idnew_scrape_item = scrape_item.create_child(web_url)self.create_task(self.run(new_scrape_item))diff --git a/cyberdrop_dl/crawlers/twpornstars.py b/cyberdrop_dl/crawlers/twpornstars.pyindex 1715dad10..398bbffde 100644--- a/cyberdrop_dl/crawlers/twpornstars.py+++ b/cyberdrop_dl/crawlers/twpornstars.py@@ -59,10 +59,10 @@ class TwPornstarsCrawler(TwimgCrawler):returnsoup = await self.request_soup(scrape_item.url)if url := soup.select_one(_SELECTORS.PHOTO):- url = self.parse_url(css.get_attr(url, "src").replace(":large", ""))+ url = self.parse_url(css.attr(url, "src").replace(":large", ""))await self.photo(scrape_item, url)elif url := soup.select_one(_SELECTORS.VIDEO):- url = self.parse_url(css.get_attr(url, "src")).with_query(None)+ url = self.parse_url(css.attr(url, "src")).with_query(None)await self.direct_file(scrape_item, url)else:raise ScrapeError(404)diff --git a/cyberdrop_dl/crawlers/xenforo/titsintops.py b/cyberdrop_dl/crawlers/xenforo/titsintops.pyindex 0d6932419..109b4db65 100644--- a/cyberdrop_dl/crawlers/xenforo/titsintops.py+++ b/cyberdrop_dl/crawlers/xenforo/titsintops.py@@ -20,10 +20,10 @@ class TitsInTopsCrawler(XenforoCrawler):return super().parse_url(fix_link(link))def is_username_or_attachment(self, link_obj: Tag) -> bool:- text = css.get_text(link_obj)+ text = css.text(link_obj)if "view attachment" in text.lower():return True- title = css.get_attr_or_none(link_obj, "title")+ title = css.attr_or_none(link_obj, "title")if title and "permanent link" in title.lower():return Truereturn super().is_username_or_attachment(link_obj)diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.pyindex dae4dab15..f1af7a62b 100644--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py@@ -149,7 +149,7 @@ def parse_login_form(resp_text: str) -> dict[str, str]:data = {name: valuefor elem in inputs- if (name := css.get_attr_or_none(elem, "name")) and (value := css.get_attr_or_none(elem, "value"))+ if (name := css.attr_or_none(elem, "name")) and (value := css.attr_or_none(elem, "value"))}if data:return datadiff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.pyindex a3ffe1c36..43e9ca6aa 100644--- a/cyberdrop_dl/crawlers/xvideos.py+++ b/cyberdrop_dl/crawlers/xvideos.py@@ -114,10 +114,10 @@ class XVideosCrawler(Crawler):soup = await self._get_soup(scrape_item.url)if error := soup.select_one(Selectors.DELETED_VIDEO):- raise ScrapeError(404, css.get_text(error))+ raise ScrapeError(404, css.text(error))title = css.page_title(soup, self.DOMAIN)- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])script = css.select_text(soup, Selectors.HLS_VIDEO_JS)m3u8_url = self.parse_url(get_text_between(script, "setVideoHLS('", "')"))m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)@@ -174,7 +174,7 @@ class XVideosCrawler(Crawler):title_tag = css.select(soup, Selectors.GALLERY_TITLE)for tag in title_tag.select("*"):tag.decompose()- title = self.create_title(css.get_text(title_tag).split(">", 1)[-1].strip(), album_id)+ title = self.create_title(css.text(title_tag).split(">", 1)[-1].strip(), album_id)scrape_item.setup_as_album(title, album_id=album_id)for _, src in self.iter_tags(soup, Selectors.GALLERY_IMG, results=results):diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 1a00ca991..587466931 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -1,23 +1,16 @@from __future__ import annotations-import functools+import htmlimport json-from typing import TYPE_CHECKING, Any, NamedTuple, ParamSpec, TypeVar, overload--import bs4.css-from bs4 import BeautifulSoup+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast, overloadfrom cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.utils.logger import log_debugif TYPE_CHECKING:- from collections.abc import Callable, Generator+ from collections.abc import Generatorfrom bs4.element import Tag- _P = ParamSpec("_P")- _R = TypeVar("_R")-class SelectorError(ScrapeError):def __init__(self, message: str | None = None) -> None:@@ -31,22 +24,27 @@ class CssAttributeSelector(NamedTuple):def __call__(self, soup: Tag) -> str:return select(soup, self.element, self.attribute)+ def text(self, tag: Tag) -> str:+ return select_text(tag, self.element)+++class JsonLD(dict[str, Any]):+ @overload+ def __getitem__(self, key: Literal["uploadDate"], /) -> str: ... # pyright: ignore[reportNoOverloadImplementation]-def not_none(func: Callable[_P, _R | None]) -> Callable[_P, _R]:- @functools.wraps(func)- def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:- result = func(*args, **kwargs)- if result is None:- raise SelectorError- return result+ @overload+ def __getitem__(self, key: str, /) -> Any:+ """Return self[key]."""- return wrapper+ __getitem__ = dict.__getitem__-@not_none-def _select_one(tag: Tag, selector: str) -> Tag | None:+def _select_one(tag: Tag, selector: str) -> Tag:"""Same as `tag.select_one` but asserts the result is not `None`"""- return tag.select_one(selector)+ result = tag.select_one(selector)+ if result is None:+ raise SelectorError(f"{selector} tag not found")+ return resultdef select_text(tag: Tag, selector: str, strip: bool = True, *, decompose: str | None = None) -> str:@@ -55,10 +53,10 @@ def select_text(tag: Tag, selector: str, strip: bool = True, *, decompose: str |if decompose:for trash in iselect(inner_tag, decompose):trash.decompose()- return get_text(inner_tag, strip)+ return text(inner_tag, strip)-def get_attr_or_none(tag: Tag, attribute: str) -> str | None:+def attr_or_none(tag: Tag, attribute: str) -> str | None:"""Same as `tag.get(attribute)` but asserts the result is a single str"""attribute_ = attributeif attribute_ == "srcset":@@ -75,14 +73,16 @@ def get_attr_or_none(tag: Tag, attribute: str) -> str | None:return value-def get_text(tag: Tag, strip: bool = True) -> str:- return tag.get_text(strip=strip)+def attr(tag: Tag, attribute: str) -> str:+ """Same as `tag.get(attribute)` but asserts the result is not `None` and is a single string"""+ result = attr_or_none(tag, attribute)+ if result is None:+ raise SelectorError(f"{attribute = } not found")+ return result-@not_none-def get_attr(tag: Tag, attribute: str) -> str | None:- """Same as `tag.get(attribute)` but asserts the result is not `None` and is a single string"""- return get_attr_or_none(tag, attribute)+def text(tag: Tag, strip: bool = True) -> str:+ return tag.get_text(strip=strip)@overload@@ -97,12 +97,7 @@ def select(tag: Tag, selector: str, attribute: str | None = None) -> Tag | str:inner_tag = _select_one(tag, selector)if not attribute:return inner_tag- return get_attr(inner_tag, attribute)---def select_one_get_attr_or_none(tag: Tag, selector: str, attribute: str) -> str | None:- if inner_tag := tag.select_one(selector):- return get_attr_or_none(inner_tag, attribute)+ return attr(inner_tag, attribute)@overload@@ -115,13 +110,13 @@ def iselect(tag: Tag, selector: str, attribute: str) -> Generator[str]: ...def iselect(tag: Tag, selector: str, attribute: str | None = None) -> Generator[Tag] | Generator[str]:"""Same as `tag.select(selector)`, but it returns a generator instead of a list."""- tags = bs4.css.CSS(tag).iselect(selector)+ tags = tag.css.iselect(selector)if not attribute:yield from tagselse:for inner_tag in tags:- if attr := get_attr_or_none(inner_tag, attribute):+ if attr := attr_or_none(inner_tag, attribute):yield attr@@ -135,111 +130,42 @@ def decompose(tag: Tag, selector: str) -> None:inner_tag.decompose()-def sanitize_page_title(title: str, domain: str) -> str:- sld = domain.rsplit(".", 1)[0].casefold()+def rstrip_domain(title: str, domain: str) -> str:+ assert domain+ second_level_domain = domain.rsplit(".", 1)[0].casefold().removeprefix("www.")- def clean(string: str, char: str):- if char in string:- front, _, tail = string.rpartition(char)- if sld in tail.casefold():- string = front.strip()+ def sanitize(string: str, char: str) -> str:+ front, _, tail = string.rpartition(char)+ if front and second_level_domain in tail.casefold():+ return front.strip()return string- return clean(clean(title, "|"), " - ")+ for char in sorted(("|", " - "), key=lambda x: title.rfind(x), reverse=True):+ title = sanitize(title, char)++ return titledef page_title(soup: Tag, domain: str | None = None) -> str:title = select_text(soup, "title")if domain:- return sanitize_page_title(title, domain)+ return rstrip_domain(title, domain)return title-def get_json_ld_date(soup: Tag) -> str:- return get_json_ld(soup)["uploadDate"]---def get_json_ld(soup: Tag, /, contains: str | None = None) -> dict[str, Any]:+def json_ld(soup: Tag, /, contains: str | None = None) -> JsonLD:selector = "script[type='application/ld+json']"if contains:selector += f":-soup-contains('{contains}')"ld_json = json.loads(select_text(soup, selector)) or {}if isinstance(ld_json, list):- return ld_json[0]-- return ld_json---def get_nuxt_data(soup: Tag) -> list[Any]:- return json.loads(select_text(soup, "script#__NUXT_DATA__"))---def parse_nuxt_obj(nuxt_data: list[Any], *attributes: str) -> dict[str, Any]:- """Parses a single object from a NUXT rich JSON payload response (__NUXT_DATA__)-- It iterates over each object until it finds an object with the desired attributes"""- if obj := next(parse_nuxt_objs(nuxt_data, *attributes), None):- return obj- raise SelectorError(f"Unable to find object with {attributes = } in NUXT_DATA")---def parse_nuxt_objs(nuxt_data: list[Any], *attributes: str) -> Generator[dict[str, Any]]:- """- Iterates over each object from a NUXT rich JSON payload response (__NUXT_DATA__)-- It bypasses the devalue parsing logic by ignoring objects without the desired attributes-- https://github.com/nuxt/nuxt/discussions/20879- """- assert attributes- first_key = attributes[0]- objects = (obj for obj in nuxt_data if isinstance(obj, dict) and all(key in obj for key in attributes))- for obj in objects:- try:- index: int = obj[first_key]- index_map: dict[str, int] = nuxt_data[index]- assert isinstance(index_map, dict)- except (LookupError, AssertionError):- index_map = obj-- yield _parse_nuxt_obj(nuxt_data, index_map)---def _parse_nuxt_obj(nuxt_data: list[Any], index_map: dict[str, int]) -> dict[str, Any]:- def hydrate(value: Any) -> Any:- if isinstance(value, list):- match value:- case ["BigInt", val]:- return int(val)- case ["Date" | "Object" | "RegExp", val, *_]:- return val- case ["Set", *values]:- return [hydrate(nuxt_data[idx]) for idx in values]- case ["Map", *values]:- return hydrate(dict(zip(*(iter(values),) * 2, strict=True)))- case ["ShallowRef" | "ShallowReactive" | "Ref" | "Reactive" | "NuxtError", idx]:- return hydrate(nuxt_data[idx])- case [str(name), *rest]:- log_debug(f"Unable to parse custom object {name} {rest}", 30)- return None- case _:- return [hydrate(nuxt_data[idx]) for idx in value]-- if isinstance(value, dict):- return _parse_nuxt_obj(nuxt_data, value)-- return value-- return {name: hydrate(nuxt_data[idx]) for name, idx in index_map.items()}-+ ld_json = ld_json[0]-def unescape(html: str) -> str:- return make_soup(html).get_text()+ return cast("JsonLD", ld_json)-def make_soup(html: str) -> BeautifulSoup:- return BeautifulSoup(html, "html.parser")+unescape = html.unescapeiframes = CssAttributeSelector("iframe", "src")diff --git a/cyberdrop_dl/utils/nuxt.py b/cyberdrop_dl/utils/nuxt.pynew file mode 100644index 000000000..4377e220f--- /dev/null+++ b/cyberdrop_dl/utils/nuxt.py@@ -0,0 +1,79 @@+from __future__ import annotations++import json+import logging+from typing import TYPE_CHECKING, Any++from cyberdrop_dl.utils import css++if TYPE_CHECKING:+ from collections.abc import Generator++ from bs4.element import Tag+++logger = logging.getLogger(__name__)+++def extract(soup: Tag) -> list[Any]:+ return json.loads(css.select_text(soup, "script#__NUXT_DATA__"))+++def parse_obj(nuxt_data: list[Any], attribute: str, *attributes: str) -> dict[str, Any]:+ """Parses a single object from a NUXT rich JSON payload response (__NUXT_DATA__)++ It iterates over each object until it finds an object with the desired attributes"""+ if obj := next(parse_objs(nuxt_data, attribute, *attributes), None):+ return obj+ raise css.SelectorError(f"Unable to find object with {attributes = } in NUXT_DATA")+++def parse_objs(nuxt_data: list[Any], attribute: str, *attributes: str) -> Generator[dict[str, Any]]:+ """+ Iterates over each object from a NUXT rich JSON payload response (__NUXT_DATA__)++ It bypasses the devalue parsing logic by ignoring objects without the desired attributes++ https://github.com/nuxt/nuxt/discussions/20879+ """+ attributes = attribute, *attributes+ objects = (obj for obj in nuxt_data if isinstance(obj, dict) and all(key in obj for key in attributes))+ for obj in objects:+ try:+ index: int = obj[attribute]+ index_map: dict[str, int] = nuxt_data[index]+ if not isinstance(index_map, dict):+ raise LookupError++ except LookupError:+ index_map = obj++ yield _parse_obj(nuxt_data, index_map)+++def _parse_obj(nuxt_data: list[Any], index_map: dict[str, int]) -> dict[str, Any]:+ def hydrate(value: Any) -> Any:+ if isinstance(value, list):+ match value:+ case ["BigInt", val]:+ return int(val)+ case ["Date" | "Object" | "RegExp", val, *_]:+ return val+ case ["Set", *values]:+ return [hydrate(nuxt_data[idx]) for idx in values]+ case ["Map", *values]:+ return hydrate(dict(zip(*(iter(values),) * 2, strict=True)))+ case ["ShallowRef" | "ShallowReactive" | "Ref" | "Reactive" | "NuxtError", idx]:+ return hydrate(nuxt_data[idx])+ case [str(name), *rest]:+ logger.warning(f"Unable to parse custom object {name} {rest}")+ return None+ case _:+ return [hydrate(nuxt_data[idx]) for idx in value]++ if isinstance(value, dict):+ return _parse_obj(nuxt_data, value)++ return value++ return {name: hydrate(nuxt_data[idx]) for name, idx in index_map.items()}diff --git a/tests/test_css.py b/tests/test_css.pynew file mode 100644index 000000000..1ba0a94e9--- /dev/null+++ b/tests/test_css.py@@ -0,0 +1,34 @@+import pytest++from cyberdrop_dl.utils import css+++ "raw, domain, expected",+ [+ ("cyberdrop-dl | GitHub", "github.com", "cyberdrop-dl"),+ ("cyberdrop-dl - GitHub", "github.com", "cyberdrop-dl"),+ ("cyberdrop-dl | GitHub - Bar", "github.com", "cyberdrop-dl"),+ ("", "github.com", ""),+ (" ", "github.com", " "),+ # case-insensitive+ ("cyberdrop-dl | GITHUB", "GitHub.com", "cyberdrop-dl"),+ ("cyberdrop-dl - GiThUb", "GITHUB.io", "cyberdrop-dl"),+ # sub-domains+ ("News | www.github.co", "www.github.co.uk", "News"),+ # no match -> unchanged+ ("cyberdrop-dl | Foo", "github.com", "cyberdrop-dl | Foo"),+ ("cyberdrop-dl - Foo", "github.com", "cyberdrop-dl - Foo"),+ # clean up once+ ("cyberdrop-dl | Foo - GitHub", "github.com", "cyberdrop-dl | Foo"),+ ("A | B | GitHub", "github.com", "A | B"),+ ("A - B - GitHub", "github.com", "A - B"),+ ],+)+def test_rstrip_domain(raw: str, domain: str, expected: str) -> None:+ assert css.rstrip_domain(raw, domain) == expected+++def test_no_domain_raise_error() -> None:+ with pytest.raises(AssertionError):+ css.rstrip_domain("cyberdrop-dl | Foo", "")diff --git a/tests/test_nextjs.py b/tests/test_nextjs.pyindex 916dece9b..cfb5728aa 100644--- a/tests/test_nextjs.py+++ b/tests/test_nextjs.py@@ -4,14 +4,14 @@ import aiohttpimport pytestfrom bs4 import BeautifulSoup-from cyberdrop_dl.utils import css, next_js+from cyberdrop_dl.utils import next_js@pytest.fixture(name="soup", scope="module")async def onepace_soup() -> AsyncGenerator[BeautifulSoup]:async with aiohttp.ClientSession() as session:resp = await session.get("https://onepace.net/en/watch")- yield css.make_soup(await resp.text())+ yield BeautifulSoup(await resp.text(), "html.parser")def test_next_js_parser(soup: BeautifulSoup) -> None: