Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: rework css module (#31)

NTFSvolumec90ebdfc54bc
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index f499a1a56..55db0ff70 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -207,7 +207,7 @@ class CheveretoCrawler(Crawler, is_generic=True):
def _get_album_files(self, soup: BeautifulSoup) -> Generator[tuple[AbsoluteHttpURL, AbsoluteHttpURL]]:
for item in soup.select(".list-item[data-object]"):
web_url = self.parse_url(css.select(item, "a.image-container", "href"))
- encoded_data = css.get_attr(item, "data-object")
+ encoded_data = css.attr(item, "data-object")
data = json.loads(urllib.parse.unquote(encoded_data))
src_url = self.parse_url(data["image"]["url"])
yield self.transform_url(web_url), src_url
diff --git a/cyberdrop_dl/crawlers/_fluid_player.py b/cyberdrop_dl/crawlers/_fluid_player.py
index 7089dc4ed..824c34c79 100644
--- a/cyberdrop_dl/crawlers/_fluid_player.py
+++ b/cyberdrop_dl/crawlers/_fluid_player.py
@@ -39,7 +39,7 @@ class FluidPlayerCrawler(Crawler, is_abc=True):
link = self.parse_url(best_format.link_str)
filename, ext = self.get_filename_and_ext(link.name)
title = open_graph.title(soup)
- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))
+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])
custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=best_format.resolution)
return await self.handle_file(
scrape_item.url, scrape_item, filename, ext, custom_filename=custom_filename, debrid_link=link
@@ -68,8 +68,8 @@ def _get_best_format(soup: BeautifulSoup) -> Format:
def parse():
for src in soup.select(Selector.VIDEO_SRC):
- url = css.get_attr(src, "src")
- quality = css.get_attr_or_none(src, "title")
+ url = css.attr(src, "src")
+ quality = css.attr_or_none(src, "title")
resolution = parse_resolution(quality)
yield Format(resolution, url)
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index 9f28cc2a7..cb9b7f6fa 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -88,7 +88,7 @@ class ForumPost:
except Exception:
date = None
- id_str = css.get_attr(article, selectors.id.attribute)
+ id_str = css.attr(article, selectors.id.attribute)
post_id = int(id_str.rsplit("-", 1)[-1])
return ForumPost(post_id, date, article, content)
@@ -532,14 +532,17 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
def _lazy_load_embeds(self, post: ForumPostProtocol) -> Iterable[str]:
selector = self.SELECTORS.posts.lazy_load_embeds
for lazy_media in css.iselect(post.content, selector.element):
- yield get_text_between(css.get_attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")
+ yield get_text_between(css.attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")
async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:
async for soup in self._web_pager(scrape_item.url, self.get_next_page):
yield soup
def get_next_page(self, soup: BeautifulSoup) -> str | None:
- return css.select_one_get_attr_or_none(soup, *self.SELECTORS.next_page)
+ try:
+ return css.select(soup, *self.SELECTORS.next_page)
+ except css.SelectorError:
+ return
@final
@error_handling_wrapper
@@ -579,7 +582,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
if not confirm_button:
return
- link_str: str = css.get_attr(confirm_button, selector.attribute)
+ link_str: str = css.attr(confirm_button, selector.attribute)
link_str = link_str.split('" class="link link--internal', 1)[0]
new_link = self.parse_url(link_str)
return await self.get_absolute_link(new_link)
@@ -596,7 +599,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
if link_obj.select_one(".username"):
return True
try:
- if link_str := css.get_attr(link_obj, self.SELECTORS.posts.links.element):
+ if link_str := css.attr(link_obj, self.SELECTORS.posts.links.element):
return self.is_attachment(link_str)
except Exception:
pass
@@ -606,7 +609,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
def iter_links(links: Iterable[Tag], attribute: str) -> Iterable[str]:
for link_tag in links:
try:
- yield css.get_attr(link_tag, attribute)
+ yield css.attr(link_tag, attribute)
except Exception:
continue
@@ -674,7 +677,7 @@ def get_post_title(soup: BeautifulSoup, selectors: MessageBoardSelectors) -> str
except (AttributeError, AssertionError, css.SelectorError) as e:
raise ScrapeError(429, message="Invalid response from forum. You may have been rate limited") from e
- if title := " ".join(css.get_text(title_block).split()):
+ if title := " ".join(css.text(title_block).split()):
return title
raise ScrapeError(422)
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.py
index c88442786..812c548aa 100644
--- a/cyberdrop_dl/crawlers/_kvs.py
+++ b/cyberdrop_dl/crawlers/_kvs.py
@@ -129,7 +129,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
filename, ext = self.get_filename_and_ext(video.url.name)
custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=video.resolution)
try:
- date_str = css.get_json_ld_date(soup)
+ date_str = css.json_ld(soup)["uploadDate"]
scrape_item.possible_datetime = self.parse_iso_date(date_str)
except (LookupError, ValueError, css.SelectorError):
date_str = css.select_text(soup, _SELECTORS.DATE).split(":", 1)[-1].strip()
@@ -173,7 +173,7 @@ def extract_kvs_video(cls: Crawler, soup: BeautifulSoup) -> KVSVideo:
if not video.title:
title = open_graph.get_title(soup) or css.page_title(soup)
assert title
- video.title = css.sanitize_page_title(title, cls.DOMAIN)
+ video.title = css.rstrip_domain(title, cls.DOMAIN)
return video
diff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.py
index b0e84dfd9..c3bb54ed8 100644
--- a/cyberdrop_dl/crawlers/_yetishare.py
+++ b/cyberdrop_dl/crawlers/_yetishare.py
@@ -109,8 +109,8 @@ class YetiShareCrawler(Crawler, is_abc=True):
total_pages = int(css.select(ajax_soup, Selector.FOLDER_TOTAL_PAGES, "value"))
for file in ajax_soup.select(Selector.FILES):
- file_url = self.parse_url(css.get_attr(file, "dtfullurl"))
- content_id = int(css.get_attr(file, "fileid"))
+ file_url = self.parse_url(css.attr(file, "dtfullurl"))
+ content_id = int(css.attr(file, "fileid"))
new_scrape_item = scrape_item.create_child(file_url)
new_scrape_item.part_of_album = not is_shared
self.create_task(self._handle_content_id_task(new_scrape_item, content_id))
@@ -157,7 +157,7 @@ class YetiShareCrawler(Crawler, is_abc=True):
# Manually parse link. Some URLs are invalid. ex: https://cyberfile.me/7cfu
# For the download URL, the slug does not actually matter. It can be anything
- raw_link = get_text_between(css.get_attr(download_tag, "onclick"), "('", "');")
+ raw_link = get_text_between(css.attr(download_tag, "onclick"), "('", "');")
token = raw_link.rpartition("?download_token=")[-1]
link = self.parse_url(raw_link).with_query(download_token=token)
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.py
index 46aa019c9..7136ff311 100644
--- a/cyberdrop_dl/crawlers/ashemaletube.py
+++ b/cyberdrop_dl/crawlers/ashemaletube.py
@@ -154,13 +154,13 @@ class AShemaleTubeCrawler(Crawler):
@error_handling_wrapper
async def proccess_image(self, scrape_item: ScrapeItem, img_tag: Tag) -> None:
if image := img_tag.select_one("img"):
- link_str: str = css.get_attr(image, "src")
+ link_str: str = css.attr(image, "src")
else:
style: str = css.select(img_tag, "a", "style")
link_str = get_text_between(style, "url('", "');")
url = self.parse_url(link_str).with_query(None)
filename, ext = self.get_filename_and_ext(url.name)
- custom_filename = self.create_custom_filename(filename, ext, file_id=css.get_attr(img_tag, "data-image-id"))
+ custom_filename = self.create_custom_filename(filename, ext, file_id=css.attr(img_tag, "data-image-id"))
await self.handle_file(url, scrape_item, filename, ext, custom_filename=custom_filename)
@error_handling_wrapper
@@ -183,7 +183,7 @@ class AShemaleTubeCrawler(Crawler):
debrid_link = best_format.url
if video_object := soup.select_one(_SELECTORS.VIDEO_PROPS_JS):
- json_data = json.loads(css.get_text(video_object))
+ json_data = json.loads(css.text(video_object))
scrape_item.possible_datetime = self.parse_iso_date(json_data.get("uploadDate", ""))
title = css.select_text(soup, "title").split("- aShemaletube.com")[0].strip()
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.py
index bc7d617cd..603199ba0 100644
--- a/cyberdrop_dl/crawlers/bunkrr.py
+++ b/cyberdrop_dl/crawlers/bunkrr.py
@@ -205,7 +205,7 @@ class BunkrrCrawler(Crawler):
soup = await self._request_soup_lenient(scrape_item.url)
src = None
if image := soup.select_one(Selector.IMAGE_PREVIEW):
- src = self.parse_url(css.get_attr(image, "src"))
+ src = self.parse_url(css.attr(image, "src"))
if len(src.parts) > 2:
src = None
diff --git a/cyberdrop_dl/crawlers/camwhores_dot_tv.py b/cyberdrop_dl/crawlers/camwhores_dot_tv.py
index 43a87b654..24ccf254a 100644
--- a/cyberdrop_dl/crawlers/camwhores_dot_tv.py
+++ b/cyberdrop_dl/crawlers/camwhores_dot_tv.py
@@ -53,7 +53,7 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler):
soup = await self.request_soup(url)
try:
- last_page = int(css.get_text(soup.select(LAST_PAGE_SELECTOR)[-1]))
+ last_page = int(css.text(soup.select(LAST_PAGE_SELECTOR)[-1]))
except IndexError:
return
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 9a62316c0..6ba0dc823 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -8,7 +8,7 @@ import re
from abc import ABC, abstractmethod
from collections import Counter
from dataclasses import dataclass, field
-from functools import partial, wraps
+from functools import wraps
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Literal, NamedTuple, ParamSpec, TypeAlias, TypeVar, final
@@ -652,7 +652,7 @@ class Crawler(ABC):
seen: set[str] = set()
for tag in css.iselect(soup, selector):
- link_str: str | None = css.get_attr_or_none(tag, attribute)
+ link_str: str | None = css.attr_or_none(tag, attribute)
if not link_str or link_str in seen:
continue
seen.add(link_str)
@@ -660,7 +660,7 @@ class Crawler(ABC):
if self.check_album_results(link, album_results):
continue
if t_tag := tag.select_one("img"):
- thumb_str: str | None = css.get_attr_or_none(t_tag, "src")
+ thumb_str: str | None = css.attr_or_none(t_tag, "src")
else:
thumb_str = None
thumb = self.parse_url(thumb_str) if thumb_str and not is_blob_or_svg(thumb_str) else None
@@ -746,8 +746,12 @@ class Crawler(ABC):
else:
selector = selector or self.NEXT_PAGE_SELECTOR
assert selector, f"No selector was provided and {self.DOMAIN} does define a next_page_selector"
- func = css.select_one_get_attr_or_none
- get_next_page = partial(func, selector=selector, attribute="href")
+
+ def get_next_page(soup: BeautifulSoup, /) -> str | None:
+ try:
+ return css.select(soup, selector, "href")
+ except css.SelectorError:
+ return
while True:
soup = await self.request_soup(page_url, impersonate=cffi or None)
diff --git a/cyberdrop_dl/crawlers/desivideo.py b/cyberdrop_dl/crawlers/desivideo.py
index e9d6ad4f0..a37611992 100644
--- a/cyberdrop_dl/crawlers/desivideo.py
+++ b/cyberdrop_dl/crawlers/desivideo.py
@@ -46,7 +46,7 @@ class DesiVideoCrawler(Crawler):
video_url = self.parse_url(Selector.VIDEO_SRC(soup))
title = css.select_text(soup, Selector.TITLE)
_, ext = self.get_filename_and_ext(video_url.name)
- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))
+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])
custom_filename = self.create_custom_filename(title, ext, file_id=video_id)
return await self.handle_file(video_url, scrape_item, video_url.name, ext, custom_filename=custom_filename)
diff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.py
index 52093e02b..4bddcb89f 100644
--- a/cyberdrop_dl/crawlers/dirtyship.py
+++ b/cyberdrop_dl/crawlers/dirtyship.py
@@ -68,7 +68,7 @@ class DirtyShipCrawler(Crawler):
if not scrape_item.url.suffix == ".jpg":
soup = await self.request_soup(scrape_item.url)
url = self.parse_url(
- next(css.get_attr(a, "href") for a in soup.select(_SELECTORS.SINGLE_PHOTO) if "full" in a.get_text())
+ next(css.attr(a, "href") for a in soup.select(_SELECTORS.SINGLE_PHOTO) if "full" in a.get_text())
)
else:
url = scrape_item.url
@@ -96,9 +96,9 @@ class DirtyShipCrawler(Crawler):
for img in thumbnails:
url = (
- css.get_attr(img, "src")
+ css.attr(img, "src")
if img.get("decoding") == "async"
- else get_highest_resolution_picture(css.get_attr(img, "srcset"))
+ else get_highest_resolution_picture(css.attr(img, "srcset"))
)
if not url:
raise ScrapeError(404)
@@ -129,10 +129,10 @@ class DirtyShipCrawler(Crawler):
def get_formats():
for video in videos:
- link_str: str = css.get_attr(video, "src")
+ link_str: str = css.attr(video, "src")
if link_str.startswith("type="):
continue
- res: str = css.get_attr(video, "title")
+ res: str = css.attr(video, "title")
link = self.parse_url(link_str)
yield (Format(int(res), link))
@@ -149,7 +149,7 @@ class DirtyShipCrawler(Crawler):
def get_flowplayer_sources(self, soup: BeautifulSoup) -> set[Format]:
flow_player = soup.select_one(_SELECTORS.FLOWPLAYER_VIDEO)
- data_item: str | None = css.get_attr_or_none(flow_player, "data-item") if flow_player else None
+ data_item: str | None = css.attr_or_none(flow_player, "data-item") if flow_player else None
if not data_item:
return set()
data_item = data_item.replace(r"\/", "/")
diff --git a/cyberdrop_dl/crawlers/eightmuses.py b/cyberdrop_dl/crawlers/eightmuses.py
index 4fd2ae7fc..cf686e9d1 100644
--- a/cyberdrop_dl/crawlers/eightmuses.py
+++ b/cyberdrop_dl/crawlers/eightmuses.py
@@ -46,8 +46,8 @@ class EightMusesCrawler(Crawler):
results = await self.get_album_results(album_id)
for tile in soup.select(TILE_SELECTOR):
- tile_link = self.parse_url(css.get_attr(tile, "href"))
- tile_title: str = css.get_attr_or_none(tile, "title") or ""
+ tile_link = self.parse_url(css.attr(tile, "href"))
+ tile_title: str = css.attr_or_none(tile, "title") or ""
image = css.select(tile, IMAGE_SELECTOR)
is_new_album = image["itemtype"] == "https://schema.org/ImageGallery"
new_album_id = f"{scrape_item.album_id}/{tile_title.replace(' ', '-')}"
diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.py
index 380fde314..260ce34ad 100644
--- a/cyberdrop_dl/crawlers/eporner.py
+++ b/cyberdrop_dl/crawlers/eporner.py
@@ -62,7 +62,7 @@ class VideoSource:
@staticmethod
def parse(tag: Tag) -> VideoSource:
- link_str: str = css.get_attr(tag, "href")
+ link_str: str = css.attr(tag, "href")
name = tag.get_text(strip=True).removeprefix("Download")
details = name.split("(", 1)[1].removesuffix(")").split(",")
res, codec, size = [d.strip() for d in details]
diff --git a/cyberdrop_dl/crawlers/fapello.py b/cyberdrop_dl/crawlers/fapello.py
index f77463ca4..0f44fed5c 100644
--- a/cyberdrop_dl/crawlers/fapello.py
+++ b/cyberdrop_dl/crawlers/fapello.py
@@ -45,7 +45,7 @@ class FapelloCrawler(Crawler):
scrape_item.setup_as_album(title)
for post in soup.select(CONTENT_SELECTOR):
- link_str: str = css.get_attr(post, "href")
+ link_str: str = css.attr(post, "href")
if "javascript" in link_str:
link_str = css.select(post, "iframe", "src")
diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.py
index 774dcf40d..0bfc4daa7 100644
--- a/cyberdrop_dl/crawlers/filester.py
+++ b/cyberdrop_dl/crawlers/filester.py
@@ -54,7 +54,7 @@ class FilesterCrawler(Crawler):
while True:
for row in soup.select(Selector.FILES):
- url = get_text_between(css.get_attr(row, "onclick"), "'", "'")
+ url = get_text_between(css.attr(row, "onclick"), "'", "'")
web_url = self.parse_url(url)
new_scrape_item = scrape_item.create_child(web_url)
self.create_task(self.run(new_scrape_item))
diff --git a/cyberdrop_dl/crawlers/fsiblog.py b/cyberdrop_dl/crawlers/fsiblog.py
index 9f4e6153d..66f511a3a 100644
--- a/cyberdrop_dl/crawlers/fsiblog.py
+++ b/cyberdrop_dl/crawlers/fsiblog.py
@@ -55,7 +55,7 @@ class FSIBlogCrawler(Crawler):
@error_handling_wrapper
async def post(self, scrape_item: ScrapeItem) -> None:
soup = await self.request_soup(scrape_item.url)
- meta: dict[str, str] = css.get_json_ld(soup)["@graph"][0]
+ meta: dict[str, str] = css.json_ld(soup)["@graph"][0]
name = meta["name"].rpartition("-")[0]
scrape_item.possible_datetime = date = self.parse_iso_date(meta["datePublished"])
title = self.create_separate_post_title(name, None, date)
diff --git a/cyberdrop_dl/crawlers/hianime.py b/cyberdrop_dl/crawlers/hianime.py
index 8d103b197..43969c3c0 100644
--- a/cyberdrop_dl/crawlers/hianime.py
+++ b/cyberdrop_dl/crawlers/hianime.py
@@ -40,10 +40,10 @@ class Episode:
@staticmethod
def from_tag(ep_tag: bs4.Tag) -> Episode:
return Episode(
- title=css.get_attr(ep_tag, "title"),
- number=int(css.get_attr(ep_tag, "data-number")),
- id=int(css.get_attr(ep_tag, "data-id")),
- path_qs=css.get_attr(ep_tag, "href"),
+ title=css.attr(ep_tag, "title"),
+ number=int(css.attr(ep_tag, "data-number")),
+ id=int(css.attr(ep_tag, "data-id")),
+ path_qs=css.attr(ep_tag, "href"),
)
@@ -169,5 +169,5 @@ def _parse_server_resp(html: str):
soup = bs4.BeautifulSoup(html, "html.parser")
for server_type in ("sub", "dub", "raw"):
if server_tag := soup.select_one(f"div[data-type={server_type}]:-soup-contains('HD-1')"):
- server_id = css.get_attr(server_tag, "data-id")
+ server_id = css.attr(server_tag, "data-id")
yield server_type, server_id
diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.py
index 6ff81c5de..8d92d9712 100644
--- a/cyberdrop_dl/crawlers/imagebam.py
+++ b/cyberdrop_dl/crawlers/imagebam.py
@@ -85,8 +85,9 @@ class ImageBamCrawler(Crawler):
for _, new_scrape_item in self.iter_children(scrape_item, soup, Selectors.THUMBNAILS, results=results):
self.create_task(self._image_task(new_scrape_item))
- next_page = css.select_one_get_attr_or_none(soup, Selectors.NEXT_PAGE, "href")
- if not next_page:
+ try:
+ next_page = css.select(soup, Selectors.NEXT_PAGE, "href")
+ except css.SelectorError:
break
soup = await self.request_soup(self.parse_url(next_page))
@@ -100,11 +101,11 @@ class ImageBamCrawler(Crawler):
image_tag = css.select(soup, Selectors.IMAGE)
if not scrape_item.album_id and (gallery_info := soup.select_one(Selectors.GALLERY_INFO)):
- gallery_id = self.parse_url(css.get_attr(gallery_info, "href")).name
+ gallery_id = self.parse_url(css.attr(gallery_info, "href")).name
scrape_item.album_id = gallery_id
- title = css.get_attr(image_tag, "alt")
- link = self.parse_url(css.get_attr(image_tag, "src"))
+ title = css.attr(image_tag, "alt")
+ link = self.parse_url(css.attr(image_tag, "src"))
custom_filename, ext = self.get_filename_and_ext(title)
await self.handle_file(link, scrape_item, link.name, ext, custom_filename=custom_filename)
diff --git a/cyberdrop_dl/crawlers/imgbox.py b/cyberdrop_dl/crawlers/imgbox.py
index e0a2f8088..6df62f116 100644
--- a/cyberdrop_dl/crawlers/imgbox.py
+++ b/cyberdrop_dl/crawlers/imgbox.py
@@ -47,7 +47,7 @@ class ImgBoxCrawler(Crawler):
scrape_item.setup_as_album(title, album_id=album_id)
for link in soup.select(IMAGES_SELECTOR):
- link_str: str = css.get_attr(link, "src").replace("thumbs", "images").replace("_b", "_o")
+ link_str: str = css.attr(link, "src").replace("thumbs", "images").replace("_b", "_o")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.py
index 895bc481a..7f30982d7 100644
--- a/cyberdrop_dl/crawlers/imx_to.py
+++ b/cyberdrop_dl/crawlers/imx_to.py
@@ -64,8 +64,8 @@ class ImxToCrawler(Crawler):
data={"imgContinue": "Continue+to+image+...+"},
)
image = css.select(soup, Selector.IMAGES)
- name = css.get_attr(image, "alt")
- link = self.parse_url(css.get_attr(image, "src"))
+ name = css.attr(image, "alt")
+ link = self.parse_url(css.attr(image, "src"))
filename, ext = self.get_filename_and_ext(name)
await self.handle_file(link, scrape_item, name, ext, custom_filename=filename)
diff --git a/cyberdrop_dl/crawlers/influencer_bitches.py b/cyberdrop_dl/crawlers/influencer_bitches.py
index 55e4a1dc0..e424b596d 100644
--- a/cyberdrop_dl/crawlers/influencer_bitches.py
+++ b/cyberdrop_dl/crawlers/influencer_bitches.py
@@ -58,7 +58,7 @@ class InfluencerBitchesCrawler(Crawler):
link = self.parse_url(link_str)
if self.check_album_results(link, results):
continue
- web_url = self.parse_url(css.get_attr(a_tag, "href"))
+ web_url = self.parse_url(css.attr(a_tag, "href"))
new_scrape_item = scrape_item.create_child(web_url)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, new_scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.py
index 10a11a5b3..22961e87b 100644
--- a/cyberdrop_dl/crawlers/kemono.py
+++ b/cyberdrop_dl/crawlers/kemono.py
@@ -578,7 +578,7 @@ class KemonoBaseCrawler(Crawler, is_abc=True):
PostSelectors.ATTACHMENTS,
):
for file in soup.select(selector):
- yield self.parse_url(css.get_attr(file, "href"))
+ yield self.parse_url(css.attr(file, "href"))
post = UserPost(
user_id=user_id,
diff --git a/cyberdrop_dl/crawlers/luxuretv.py b/cyberdrop_dl/crawlers/luxuretv.py
index 1c4007266..55cdfdac9 100644
--- a/cyberdrop_dl/crawlers/luxuretv.py
+++ b/cyberdrop_dl/crawlers/luxuretv.py
@@ -62,10 +62,10 @@ class LuxureTVCrawler(Crawler):
return
soup = await self.request_soup(scrape_item.url, impersonate=True)
- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))
+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])
video_player = css.select(soup, Selector.VIDEO_PLAYER)
title = css.select_text(soup, Selector.TITLE)
- link = self.parse_url(css.get_attr(video_player, "src"))
+ link = self.parse_url(css.attr(video_player, "src"))
filename, ext = self.get_filename_and_ext(link.name)
custom_filename = self.create_custom_filename(title, ext, file_id=video_id)
diff --git a/cyberdrop_dl/crawlers/masahub.py b/cyberdrop_dl/crawlers/masahub.py
index e2dca8eb1..9dda4c47f 100644
--- a/cyberdrop_dl/crawlers/masahub.py
+++ b/cyberdrop_dl/crawlers/masahub.py
@@ -45,7 +45,7 @@ class MasahubCrawler(Crawler):
link = self.parse_url(Selector.VIDEO_SRC(soup))
title = css.select_text(soup, Selector.TITLE)
_, ext = self.get_filename_and_ext(link.name)
- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))
+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])
custom_filename = self.create_custom_filename(title, ext)
return await self.handle_file(link, scrape_item, link.name, ext, custom_filename=custom_filename)
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 2565afbe3..0e580d00b 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -214,10 +214,10 @@ def _extract_download_link(soup: BeautifulSoup) -> str:
raise ScrapeError(410)
raise ScrapeError(422)
- if encoded_url := css.get_attr_or_none(download_button, "data-scrambled-url"):
+ if encoded_url := css.attr_or_none(download_button, "data-scrambled-url"):
return base64.b64decode(encoded_url).decode()
- url = css.get_attr(download_button, "href")
+ url = css.attr(download_button, "href")
if is_blob_or_svg(url):
raise ScrapeError(422)
return url
diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.py
index 2c4425eda..0f86271a0 100644
--- a/cyberdrop_dl/crawlers/missav.py
+++ b/cyberdrop_dl/crawlers/missav.py
@@ -69,7 +69,7 @@ class MissAVCrawler(Crawler):
if date_str:
scrape_item.possible_datetime = self.parse_iso_date(date_str)
elif date_tag := soup.select_one(Selector.DATE):
- scrape_item.possible_datetime = self.parse_iso_date(css.get_attr(date_tag, "datetime"))
+ scrape_item.possible_datetime = self.parse_iso_date(css.attr(date_tag, "datetime"))
else:
_ = self.parse_date("") # Trigger warning
@@ -89,7 +89,7 @@ def _get_uuid(soup: BeautifulSoup) -> str:
def _fix_title(title: str, dvd_code_tag: Tag) -> str:
- dvd_code = css.get_text(dvd_code_tag).upper()
+ dvd_code = css.text(dvd_code_tag).upper()
uncensored = "UNCENSORED" in dvd_code
leak = "LEAK" in dvd_code
for trash in ("-UNCENSORED", "-LEAK"):
diff --git a/cyberdrop_dl/crawlers/motherless.py b/cyberdrop_dl/crawlers/motherless.py
index b554fc0a3..639120366 100644
--- a/cyberdrop_dl/crawlers/motherless.py
+++ b/cyberdrop_dl/crawlers/motherless.py
@@ -172,7 +172,7 @@ class MotherlessCrawler(Crawler):
title_tag = soup.select_one(GROUP_TITLE_SELECTOR)
if title_tag:
- parent_title: str = css.get_attr(title_tag, "title") if from_gallery else title_tag.get_text(strip=True)
+ parent_title: str = css.attr(title_tag, "title") if from_gallery else title_tag.get_text(strip=True)
parent_path = parent_id if from_gallery else f"g/{parent_id}"
parent_url = PRIMARY_URL / parent_path
@@ -195,7 +195,7 @@ def check_soup(soup: BeautifulSoup) -> None:
def get_media_info(soup: BeautifulSoup) -> MediaInfo:
media_js = soup.select_one(MEDIA_INFO_JS_SELECTOR)
- js_text = css.get_text(media_js) if media_js else None
+ js_text = css.text(media_js) if media_js else None
if not js_text:
return MediaInfo("gallery", "")
media_type = js_text.split("__mediatype", 1)[-1].split("=", 1)[-1].split(",", 1)[0].strip()
diff --git a/cyberdrop_dl/crawlers/mydesi.py b/cyberdrop_dl/crawlers/mydesi.py
index cb47f8f22..3bddd98de 100644
--- a/cyberdrop_dl/crawlers/mydesi.py
+++ b/cyberdrop_dl/crawlers/mydesi.py
@@ -51,7 +51,7 @@ class MyDesiCrawler(Crawler):
resolution, link = max(_parse_formats(soup))
link = self.parse_url(link)
_, ext = self.get_filename_and_ext(link.name)
- metadata: dict[str, str] = css.get_json_ld(soup)["subjectOf"]
+ metadata: dict[str, str] = css.json_ld(soup)["subjectOf"]
title = metadata["name"]
scrape_item.possible_datetime = self.parse_iso_date(metadata.get("uploadDate", ""))
custom_filename = self.create_custom_filename(title, ext, resolution=resolution)
@@ -76,7 +76,7 @@ class MyDesiCrawler(Crawler):
def _parse_formats(soup: BeautifulSoup) -> Generator[tuple[Resolution, str]]:
for src in soup.select("#video-rate > a"):
- quality = css.get_attr(src, "title")
- link = css.get_attr(src, "href")
+ quality = css.attr(src, "title")
+ link = css.attr(src, "href")
resolution = Resolution.highest() if "original" in quality.casefold() else Resolution.parse(quality)
yield resolution, link
diff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.py
index de0c7eddb..e2d5f1800 100644
--- a/cyberdrop_dl/crawlers/odnoklassniki.py
+++ b/cyberdrop_dl/crawlers/odnoklassniki.py
@@ -76,7 +76,12 @@ class OdnoklassnikiCrawler(Crawler):
channel_id = channel_str.removeprefix("c")
gwt_hash = get_text_between(css.select_text(soup, Selector.CHANNEL_HASH), 'gwtHash:"', '",')
- last_element_id = css.select_one_get_attr_or_none(soup, *Selector.CHANNEL_LAST_ELEMENT)
+
+ try:
+ last_element_id = css.select(soup, *Selector.CHANNEL_LAST_ELEMENT)
+ except css.SelectorError:
+ last_element_id = None
+
name = css.select_text(soup, Selector.CHANNEL_NAME)
scrape_item.setup_as_album(self.create_title(name, channel_id), album_id=channel_id)
@@ -139,7 +144,7 @@ class OdnoklassnikiCrawler(Crawler):
cdn_url = self.parse_url(src)
# downloads may fail if we have cdn cookies
self.client.client_manager.cookies.clear_domain(cdn_url.host)
- json_ld = css.get_json_ld(soup)
+ json_ld = css.json_ld(soup)
title: str = metadata["movie"].get("title") or json_ld["name"]
scrape_item.possible_datetime = self.parse_iso_date(json_ld["uploadDate"])
filename = self.create_custom_filename(title, ".mp4", file_id=video_id, resolution=resolution)
diff --git a/cyberdrop_dl/crawlers/pimp_bunny.py b/cyberdrop_dl/crawlers/pimp_bunny.py
index 26cbafdf3..9d98fe505 100644
--- a/cyberdrop_dl/crawlers/pimp_bunny.py
+++ b/cyberdrop_dl/crawlers/pimp_bunny.py
@@ -128,7 +128,7 @@ class PimpBunnyCrawler(Crawler):
custom_filename = self.create_custom_filename(
video.title, video.url.suffix, file_id=video.id, resolution=video.resolution
)
- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))
+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])
await self.handle_file(
scrape_item.url,
scrape_item,
diff --git a/cyberdrop_dl/crawlers/pimpandhost.py b/cyberdrop_dl/crawlers/pimpandhost.py
index 8d824c924..4c40782a8 100644
--- a/cyberdrop_dl/crawlers/pimpandhost.py
+++ b/cyberdrop_dl/crawlers/pimpandhost.py
@@ -42,7 +42,7 @@ class PimpAndHostCrawler(Crawler):
scrape_item.setup_as_album(title, album_id=album_id)
if date_tag := soup.select_one(DATE_SELECTOR):
- scrape_item.possible_datetime = self.parse_date(css.get_attr(date_tag, "title"), DATE_FORMAT)
+ scrape_item.possible_datetime = self.parse_date(css.attr(date_tag, "title"), DATE_FORMAT)
for _, new_scrape_item in self.iter_children(scrape_item, soup, FILES_SELECTOR):
self.create_task(self.run(new_scrape_item))
diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.py
index bdbd161fa..f581332af 100644
--- a/cyberdrop_dl/crawlers/pkmncards.py
+++ b/cyberdrop_dl/crawlers/pkmncards.py
@@ -138,8 +138,8 @@ class PkmncardsCrawler(Crawler):
async for soup in self.web_pager(page_url):
for thumb in soup.select(_SELECTORS.CARD):
link_str = css.select(thumb, "img", "src")
- card_page_url_str = css.get_attr(thumb, "href")
- title = css.get_attr(thumb, "title")
+ card_page_url_str = css.attr(thumb, "href")
+ title = css.attr(thumb, "title")
card_page_url = self.parse_url(card_page_url_str)
download_url = self.parse_url(link_str)
simple_card = create_simple_card(title, download_url)
diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.py
index 3e96b3269..121961078 100644
--- a/cyberdrop_dl/crawlers/pmvhaven.py
+++ b/cyberdrop_dl/crawlers/pmvhaven.py
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css
+from cyberdrop_dl.utils import css, nuxt
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -53,14 +53,14 @@ class PMVHavenCrawler(Crawler):
title = self.create_title(title)
scrape_item.setup_as_profile(title)
- nuxt_data = css.get_nuxt_data(soup)
+ nuxt_data = nuxt.extract(soup)
await self._process_video_list(scrape_item, nuxt_data)
@error_handling_wrapper
async def playlist(self, scrape_item: ScrapeItem) -> None:
soup = await self.request_soup(scrape_item.url)
- nuxt_data = css.get_nuxt_data(soup)
- playlist = css.parse_nuxt_obj(nuxt_data, "playlist")
+ nuxt_data = nuxt.extract(soup)
+ playlist = nuxt.parse_obj(nuxt_data, "playlist")
name = playlist["name"]
title = self.create_title(f"{name} [playlist]")
scrape_item.setup_as_album(title)
@@ -72,7 +72,7 @@ class PMVHavenCrawler(Crawler):
tags = scrape_item.url.query.get("tags") or scrape_item.url.query.get("musicSong")
title = self.create_title(f"{tags} [search]")
scrape_item.setup_as_album(title)
- nuxt_data = css.get_nuxt_data(soup)
+ nuxt_data = nuxt.extract(soup)
await self._process_video_list(scrape_item, nuxt_data)
@error_handling_wrapper
@@ -81,15 +81,15 @@ class PMVHavenCrawler(Crawler):
return
soup = await self.request_soup(scrape_item.url)
- nuxt_data = css.get_nuxt_data(soup)
- video = css.parse_nuxt_obj(nuxt_data, "video", "uploaderVideosCount")
+ nuxt_data = nuxt.extract(soup)
+ video = nuxt.parse_obj(nuxt_data, "video", "uploaderVideosCount")
await self._process_video_info(scrape_item, video)
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@error_handling_wrapper
async def _process_video_list(self, scrape_item: ScrapeItem, nuxt_data: list[Any]) -> None:
- for video_info in css.parse_nuxt_objs(nuxt_data, "videoUrl"):
+ for video_info in nuxt.parse_objs(nuxt_data, "videoUrl"):
await self._process_video_info(scrape_item, video_info)
@error_handling_wrapper
diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.py
index bcafd761c..596bd0e75 100644
--- a/cyberdrop_dl/crawlers/pornhub.py
+++ b/cyberdrop_dl/crawlers/pornhub.py
@@ -205,8 +205,8 @@ class PornHubCrawler(Crawler):
link_str: str = css.select(soup, _SELECTORS.PHOTO, "src")
link = self.parse_url(link_str)
album_tag = css.select(soup, _SELECTORS.ALBUM_FROM_PHOTO)
- album_name = css.get_text(album_tag)
- album_link_str: str = css.get_attr(album_tag, "href")
+ album_name = css.text(album_tag)
+ album_link_str: str = css.attr(album_tag, "href")
album_id: str = album_link_str.split("/")[-1]
title = self.create_title(album_name, album_id)
scrape_item.setup_as_album(title, album_id=album_id)
@@ -217,7 +217,7 @@ class PornHubCrawler(Crawler):
soup = await self.request_soup(scrape_item.url)
attributes = "data-mp4", "data-fallback", "data-webm"
gif_tag = css.select(soup, _SELECTORS.GIF)
- link_str = next(value for attr in attributes if (value := css.get_attr_or_none(gif_tag, attr)))
+ link_str = next(value for attr in attributes if (value := css.attr_or_none(gif_tag, attr)))
link = self.parse_url(link_str)
await self._process_photo(scrape_item, link)
diff --git a/cyberdrop_dl/crawlers/pornpics.py b/cyberdrop_dl/crawlers/pornpics.py
index d6450f0d1..b3481b0b2 100644
--- a/cyberdrop_dl/crawlers/pornpics.py
+++ b/cyberdrop_dl/crawlers/pornpics.py
@@ -108,7 +108,7 @@ class PornPicsCrawler(Crawler):
if not offset: # offset == 0 does not return JSON
soup = await self.request_soup(current_page)
items = soup.select(IMAGE_SELECTOR)
- return soup, tuple(self.parse_url(css.get_attr(image, "href")) for image in items)
+ return soup, tuple(self.parse_url(css.attr(image, "href")) for image in items)
# The response is JSON but the "content-type" is wrong
async with self.request(current_page) as resp:
diff --git a/cyberdrop_dl/crawlers/pornzog.py b/cyberdrop_dl/crawlers/pornzog.py
index c324eaa67..e4553dfe4 100644
--- a/cyberdrop_dl/crawlers/pornzog.py
+++ b/cyberdrop_dl/crawlers/pornzog.py
@@ -37,6 +37,6 @@ class PornZogCrawler(Crawler):
async def video(self, scrape_item: ScrapeItem) -> None:
soup = await self.request_soup(scrape_item.url)
iframe = css.select(soup, Selectors.EMBED_IFRAME)
- external_url = self.parse_url(css.get_attr(iframe, "src"))
+ external_url = self.parse_url(css.attr(iframe, "src"))
new_scrape_item = scrape_item.create_child(external_url)
return self.handle_external_links(new_scrape_item)
diff --git a/cyberdrop_dl/crawlers/rule34vault.py b/cyberdrop_dl/crawlers/rule34vault.py
index 87049a653..8515cc4b3 100644
--- a/cyberdrop_dl/crawlers/rule34vault.py
+++ b/cyberdrop_dl/crawlers/rule34vault.py
@@ -83,7 +83,7 @@ class Rule34VaultCrawler(Crawler):
scrape_item.url = canonical_url
media_tag = soup.select_one(_SELECTORS.VIDEO) or soup.select_one(_SELECTORS.IMAGE)
assert media_tag
- link_str: str = css.get_attr(media_tag, "src")
+ link_str: str = css.attr(media_tag, "src")
for trash in (".small", ".thumbnail", ".picsmall", ".720", ".hevc"):
link_str = link_str.replace(trash, "")
diff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.py
index 28f899392..8863deeb7 100644
--- a/cyberdrop_dl/crawlers/rule34video.py
+++ b/cyberdrop_dl/crawlers/rule34video.py
@@ -57,7 +57,7 @@ class Rule34VideoCrawler(KernelVideoSharingCrawler):
if not title:
title_tag = css.select(soup, Selector.TITLE)
css.decompose(title_tag, "span")
- title = css.get_text(title_tag)
+ title = css.text(title_tag)
for trash in ("Videos for: ", "Tagged with "):
title = title.removeprefix(trash)
diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.py
index 2b8fee7ea..4e22b72c9 100644
--- a/cyberdrop_dl/crawlers/rumble.py
+++ b/cyberdrop_dl/crawlers/rumble.py
@@ -148,7 +148,7 @@ class RumbleCrawler(Crawler):
return
soup = await self.request_soup(scrape_item.url)
- embed_id = self.parse_url(css.get_json_ld(soup)["embedUrl"]).name
+ embed_id = self.parse_url(css.json_ld(soup)["embedUrl"]).name
await self.embed(scrape_item, embed_id)
@error_handling_wrapper
@@ -161,7 +161,7 @@ class RumbleCrawler(Crawler):
_, ext = self.get_filename_and_ext(best_format.url.name)
video_name = self.create_custom_filename(video.title, ext, file_id=embed_id, resolution=best_format.resolution)
- scrape_item.possible_datetime = self.parse_iso_date(video.upload_date)
+ scrape_item.possible_datetime = self.parse_iso_date(video["uploadDate"])
scrape_item.url = video.url
self.create_task(
self.handle_file(
diff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.py
index c85a15d79..cc964ae43 100644
--- a/cyberdrop_dl/crawlers/spankbang.py
+++ b/cyberdrop_dl/crawlers/spankbang.py
@@ -173,7 +173,7 @@ def _parse_video(soup: BeautifulSoup, display_id: str) -> Video:
url=url,
stream_id=get_text_between(stream_js_text, "ana_video_id = ", ";").strip("'"),
stream_key=css.select(soup, "[data-streamkey]", "data-streamkey"),
- title=css.get_attr_or_none(title_tag, "title") or css.get_text(title_tag),
+ title=css.attr_or_none(title_tag, "title") or css.text(title_tag),
)
diff --git a/cyberdrop_dl/crawlers/tnaflix.py b/cyberdrop_dl/crawlers/tnaflix.py
index 344738835..90f960723 100644
--- a/cyberdrop_dl/crawlers/tnaflix.py
+++ b/cyberdrop_dl/crawlers/tnaflix.py
@@ -63,7 +63,7 @@ class TNAFlixCrawler(Crawler):
link = self.parse_url(best_format.link_str)
filename, ext = self.get_filename_and_ext(link.name)
title = open_graph.title(soup)
- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))
+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])
custom_filename = self.create_custom_filename(title, ext, file_id=video_id, resolution=best_format.resolution)
return await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
@@ -83,8 +83,8 @@ class TNAFlixCrawler(Crawler):
def _get_best_format(soup: BeautifulSoup) -> Format:
def parse():
for src in soup.select(Selector.VIDEO_SRC):
- url = css.get_attr(src, "src")
- resolution = Resolution.parse(css.get_attr(src, "size"))
+ url = css.attr(src, "src")
+ resolution = Resolution.parse(css.attr(src, "size"))
yield Format(resolution, url)
return max(parse())
diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.py
index e82d55a9d..f4b6ac1bb 100644
--- a/cyberdrop_dl/crawlers/tokyomotion.py
+++ b/cyberdrop_dl/crawlers/tokyomotion.py
@@ -89,7 +89,7 @@ class TokioMotionCrawler(Crawler):
raise ScrapeError(422, "Couldn't find video source")
scrape_item.possible_datetime = self.parse_date(css.select_text(soup, _SELECTORS.VIDEO_DATE))
- link_str = css.get_attr(src, "src")
+ link_str = css.attr(src, "src")
link = self.parse_url(link_str)
title = css.select_text(soup, "title").rsplit(" - TOKYO Motion")[0].strip()
filename, ext = f"{video_id}.mp4", ".mp4"
@@ -109,7 +109,7 @@ class TokioMotionCrawler(Crawler):
raise ScrapeError(401, "Private Photo")
raise ScrapeError(422, "Couldn't find image source")
- link_str: str = css.get_attr(img, "src")
+ link_str: str = css.attr(img, "src")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/toonily.py b/cyberdrop_dl/crawlers/toonily.py
index 93142500d..a35800aa7 100644
--- a/cyberdrop_dl/crawlers/toonily.py
+++ b/cyberdrop_dl/crawlers/toonily.py
@@ -41,7 +41,7 @@ class ToonilyCrawler(Crawler):
title_tag = css.select(soup, Selector.SERIES_TITLE)
css.decompose(title_tag, "*")
- series_title = self.create_title(css.get_text(title_tag))
+ series_title = self.create_title(css.text(title_tag))
scrape_item.setup_as_profile(series_title)
for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.CHAPTER):
self.create_task(self.run(new_scrape_item))
@@ -56,7 +56,7 @@ class ToonilyCrawler(Crawler):
scrape_item.add_to_parent_title(series_title)
scrape_item.setup_as_album(chapter_title)
- iso_date = css.get_json_ld(soup)["@graph"][0]["datePublished"]
+ iso_date = css.json_ld(soup)["@graph"][0]["datePublished"]
scrape_item.possible_datetime = self.parse_iso_date(iso_date)
for _, link in self.iter_tags(soup, Selector.IMAGE, "src"):
diff --git a/cyberdrop_dl/crawlers/transflix.py b/cyberdrop_dl/crawlers/transflix.py
index 147d54bde..1919b4e9b 100644
--- a/cyberdrop_dl/crawlers/transflix.py
+++ b/cyberdrop_dl/crawlers/transflix.py
@@ -49,7 +49,7 @@ class TransflixCrawler(Crawler):
soup = await self.request_soup(scrape_item.url)
title = open_graph.title(soup)
video = css.select(soup, _SELECTORS.VIDEO)
- link = self.parse_url(css.get_attr(video, "src"))
+ link = self.parse_url(css.attr(video, "src"))
filename, ext = self.get_filename_and_ext(link.name)
scrape_item.possible_datetime = _timestamp_from_filename(link.name)
custom_filename = self.create_custom_filename(title, ext, file_id=video_id)
diff --git a/cyberdrop_dl/crawlers/turbovid.py b/cyberdrop_dl/crawlers/turbovid.py
index 46c054c50..1e701b926 100644
--- a/cyberdrop_dl/crawlers/turbovid.py
+++ b/cyberdrop_dl/crawlers/turbovid.py
@@ -61,7 +61,7 @@ class TurboVidCrawler(Crawler):
scrape_item.setup_as_album(title, album_id=album_id)
for row in soup.select(Selector.ALBUM_FILES):
- file_id = css.get_attr(row, "data-id")
+ file_id = css.attr(row, "data-id")
web_url = self.PRIMARY_URL / "d" / file_id
new_scrape_item = scrape_item.create_child(web_url)
self.create_task(self.run(new_scrape_item))
diff --git a/cyberdrop_dl/crawlers/twpornstars.py b/cyberdrop_dl/crawlers/twpornstars.py
index 1715dad10..398bbffde 100644
--- a/cyberdrop_dl/crawlers/twpornstars.py
+++ b/cyberdrop_dl/crawlers/twpornstars.py
@@ -59,10 +59,10 @@ class TwPornstarsCrawler(TwimgCrawler):
return
soup = await self.request_soup(scrape_item.url)
if url := soup.select_one(_SELECTORS.PHOTO):
- url = self.parse_url(css.get_attr(url, "src").replace(":large", ""))
+ url = self.parse_url(css.attr(url, "src").replace(":large", ""))
await self.photo(scrape_item, url)
elif url := soup.select_one(_SELECTORS.VIDEO):
- url = self.parse_url(css.get_attr(url, "src")).with_query(None)
+ url = self.parse_url(css.attr(url, "src")).with_query(None)
await self.direct_file(scrape_item, url)
else:
raise ScrapeError(404)
diff --git a/cyberdrop_dl/crawlers/xenforo/titsintops.py b/cyberdrop_dl/crawlers/xenforo/titsintops.py
index 0d6932419..109b4db65 100644
--- a/cyberdrop_dl/crawlers/xenforo/titsintops.py
+++ b/cyberdrop_dl/crawlers/xenforo/titsintops.py
@@ -20,10 +20,10 @@ class TitsInTopsCrawler(XenforoCrawler):
return super().parse_url(fix_link(link))
def is_username_or_attachment(self, link_obj: Tag) -> bool:
- text = css.get_text(link_obj)
+ text = css.text(link_obj)
if "view attachment" in text.lower():
return True
- title = css.get_attr_or_none(link_obj, "title")
+ title = css.attr_or_none(link_obj, "title")
if title and "permanent link" in title.lower():
return True
return super().is_username_or_attachment(link_obj)
diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.py
index dae4dab15..f1af7a62b 100644
--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py
+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py
@@ -149,7 +149,7 @@ def parse_login_form(resp_text: str) -> dict[str, str]:
data = {
name: value
for elem in inputs
- if (name := css.get_attr_or_none(elem, "name")) and (value := css.get_attr_or_none(elem, "value"))
+ if (name := css.attr_or_none(elem, "name")) and (value := css.attr_or_none(elem, "value"))
}
if data:
return data
diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.py
index a3ffe1c36..43e9ca6aa 100644
--- a/cyberdrop_dl/crawlers/xvideos.py
+++ b/cyberdrop_dl/crawlers/xvideos.py
@@ -114,10 +114,10 @@ class XVideosCrawler(Crawler):
soup = await self._get_soup(scrape_item.url)
if error := soup.select_one(Selectors.DELETED_VIDEO):
- raise ScrapeError(404, css.get_text(error))
+ raise ScrapeError(404, css.text(error))
title = css.page_title(soup, self.DOMAIN)
- scrape_item.possible_datetime = self.parse_iso_date(css.get_json_ld_date(soup))
+ scrape_item.possible_datetime = self.parse_iso_date(css.json_ld(soup)["uploadDate"])
script = css.select_text(soup, Selectors.HLS_VIDEO_JS)
m3u8_url = self.parse_url(get_text_between(script, "setVideoHLS('", "')"))
m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)
@@ -174,7 +174,7 @@ class XVideosCrawler(Crawler):
title_tag = css.select(soup, Selectors.GALLERY_TITLE)
for tag in title_tag.select("*"):
tag.decompose()
- title = self.create_title(css.get_text(title_tag).split(">", 1)[-1].strip(), album_id)
+ title = self.create_title(css.text(title_tag).split(">", 1)[-1].strip(), album_id)
scrape_item.setup_as_album(title, album_id=album_id)
for _, src in self.iter_tags(soup, Selectors.GALLERY_IMG, results=results):
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 1a00ca991..587466931 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -1,23 +1,16 @@
from __future__ import annotations
-import functools
+import html
import json
-from typing import TYPE_CHECKING, Any, NamedTuple, ParamSpec, TypeVar, overload
-
-import bs4.css
-from bs4 import BeautifulSoup
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast, overload
from cyberdrop_dl.exceptions import ScrapeError
-from cyberdrop_dl.utils.logger import log_debug
if TYPE_CHECKING:
- from collections.abc import Callable, Generator
+ from collections.abc import Generator
from bs4.element import Tag
- _P = ParamSpec("_P")
- _R = TypeVar("_R")
-
class SelectorError(ScrapeError):
def __init__(self, message: str | None = None) -> None:
@@ -31,22 +24,27 @@ class CssAttributeSelector(NamedTuple):
def __call__(self, soup: Tag) -> str:
return select(soup, self.element, self.attribute)
+ def text(self, tag: Tag) -> str:
+ return select_text(tag, self.element)
+
+
+class JsonLD(dict[str, Any]):
+ @overload
+ def __getitem__(self, key: Literal["uploadDate"], /) -> str: ... # pyright: ignore[reportNoOverloadImplementation]
-def not_none(func: Callable[_P, _R | None]) -> Callable[_P, _R]:
- @functools.wraps(func)
- def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
- result = func(*args, **kwargs)
- if result is None:
- raise SelectorError
- return result
+ @overload
+ def __getitem__(self, key: str, /) -> Any:
+ """Return self[key]."""
- return wrapper
+ __getitem__ = dict.__getitem__
-@not_none
-def _select_one(tag: Tag, selector: str) -> Tag | None:
+def _select_one(tag: Tag, selector: str) -> Tag:
"""Same as `tag.select_one` but asserts the result is not `None`"""
- return tag.select_one(selector)
+ result = tag.select_one(selector)
+ if result is None:
+ raise SelectorError(f"{selector} tag not found")
+ return result
def select_text(tag: Tag, selector: str, strip: bool = True, *, decompose: str | None = None) -> str:
@@ -55,10 +53,10 @@ def select_text(tag: Tag, selector: str, strip: bool = True, *, decompose: str |
if decompose:
for trash in iselect(inner_tag, decompose):
trash.decompose()
- return get_text(inner_tag, strip)
+ return text(inner_tag, strip)
-def get_attr_or_none(tag: Tag, attribute: str) -> str | None:
+def attr_or_none(tag: Tag, attribute: str) -> str | None:
"""Same as `tag.get(attribute)` but asserts the result is a single str"""
attribute_ = attribute
if attribute_ == "srcset":
@@ -75,14 +73,16 @@ def get_attr_or_none(tag: Tag, attribute: str) -> str | None:
return value
-def get_text(tag: Tag, strip: bool = True) -> str:
- return tag.get_text(strip=strip)
+def attr(tag: Tag, attribute: str) -> str:
+ """Same as `tag.get(attribute)` but asserts the result is not `None` and is a single string"""
+ result = attr_or_none(tag, attribute)
+ if result is None:
+ raise SelectorError(f"{attribute = } not found")
+ return result
-@not_none
-def get_attr(tag: Tag, attribute: str) -> str | None:
- """Same as `tag.get(attribute)` but asserts the result is not `None` and is a single string"""
- return get_attr_or_none(tag, attribute)
+def text(tag: Tag, strip: bool = True) -> str:
+ return tag.get_text(strip=strip)
@overload
@@ -97,12 +97,7 @@ def select(tag: Tag, selector: str, attribute: str | None = None) -> Tag | str:
inner_tag = _select_one(tag, selector)
if not attribute:
return inner_tag
- return get_attr(inner_tag, attribute)
-
-
-def select_one_get_attr_or_none(tag: Tag, selector: str, attribute: str) -> str | None:
- if inner_tag := tag.select_one(selector):
- return get_attr_or_none(inner_tag, attribute)
+ return attr(inner_tag, attribute)
@overload
@@ -115,13 +110,13 @@ def iselect(tag: Tag, selector: str, attribute: str) -> Generator[str]: ...
def iselect(tag: Tag, selector: str, attribute: str | None = None) -> Generator[Tag] | Generator[str]:
"""Same as `tag.select(selector)`, but it returns a generator instead of a list."""
- tags = bs4.css.CSS(tag).iselect(selector)
+ tags = tag.css.iselect(selector)
if not attribute:
yield from tags
else:
for inner_tag in tags:
- if attr := get_attr_or_none(inner_tag, attribute):
+ if attr := attr_or_none(inner_tag, attribute):
yield attr
@@ -135,111 +130,42 @@ def decompose(tag: Tag, selector: str) -> None:
inner_tag.decompose()
-def sanitize_page_title(title: str, domain: str) -> str:
- sld = domain.rsplit(".", 1)[0].casefold()
+def rstrip_domain(title: str, domain: str) -> str:
+ assert domain
+ second_level_domain = domain.rsplit(".", 1)[0].casefold().removeprefix("www.")
- def clean(string: str, char: str):
- if char in string:
- front, _, tail = string.rpartition(char)
- if sld in tail.casefold():
- string = front.strip()
+ def sanitize(string: str, char: str) -> str:
+ front, _, tail = string.rpartition(char)
+ if front and second_level_domain in tail.casefold():
+ return front.strip()
return string
- return clean(clean(title, "|"), " - ")
+ for char in sorted(("|", " - "), key=lambda x: title.rfind(x), reverse=True):
+ title = sanitize(title, char)
+
+ return title
def page_title(soup: Tag, domain: str | None = None) -> str:
title = select_text(soup, "title")
if domain:
- return sanitize_page_title(title, domain)
+ return rstrip_domain(title, domain)
return title
-def get_json_ld_date(soup: Tag) -> str:
- return get_json_ld(soup)["uploadDate"]
-
-
-def get_json_ld(soup: Tag, /, contains: str | None = None) -> dict[str, Any]:
+def json_ld(soup: Tag, /, contains: str | None = None) -> JsonLD:
selector = "script[type='application/ld+json']"
if contains:
selector += f":-soup-contains('{contains}')"
ld_json = json.loads(select_text(soup, selector)) or {}
if isinstance(ld_json, list):
- return ld_json[0]
-
- return ld_json
-
-
-def get_nuxt_data(soup: Tag) -> list[Any]:
- return json.loads(select_text(soup, "script#__NUXT_DATA__"))
-
-
-def parse_nuxt_obj(nuxt_data: list[Any], *attributes: str) -> dict[str, Any]:
- """Parses a single object from a NUXT rich JSON payload response (__NUXT_DATA__)
-
- It iterates over each object until it finds an object with the desired attributes"""
- if obj := next(parse_nuxt_objs(nuxt_data, *attributes), None):
- return obj
- raise SelectorError(f"Unable to find object with {attributes = } in NUXT_DATA")
-
-
-def parse_nuxt_objs(nuxt_data: list[Any], *attributes: str) -> Generator[dict[str, Any]]:
- """
- Iterates over each object from a NUXT rich JSON payload response (__NUXT_DATA__)
-
- It bypasses the devalue parsing logic by ignoring objects without the desired attributes
-
- https://github.com/nuxt/nuxt/discussions/20879
- """
- assert attributes
- first_key = attributes[0]
- objects = (obj for obj in nuxt_data if isinstance(obj, dict) and all(key in obj for key in attributes))
- for obj in objects:
- try:
- index: int = obj[first_key]
- index_map: dict[str, int] = nuxt_data[index]
- assert isinstance(index_map, dict)
- except (LookupError, AssertionError):
- index_map = obj
-
- yield _parse_nuxt_obj(nuxt_data, index_map)
-
-
-def _parse_nuxt_obj(nuxt_data: list[Any], index_map: dict[str, int]) -> dict[str, Any]:
- def hydrate(value: Any) -> Any:
- if isinstance(value, list):
- match value:
- case ["BigInt", val]:
- return int(val)
- case ["Date" | "Object" | "RegExp", val, *_]:
- return val
- case ["Set", *values]:
- return [hydrate(nuxt_data[idx]) for idx in values]
- case ["Map", *values]:
- return hydrate(dict(zip(*(iter(values),) * 2, strict=True)))
- case ["ShallowRef" | "ShallowReactive" | "Ref" | "Reactive" | "NuxtError", idx]:
- return hydrate(nuxt_data[idx])
- case [str(name), *rest]:
- log_debug(f"Unable to parse custom object {name} {rest}", 30)
- return None
- case _:
- return [hydrate(nuxt_data[idx]) for idx in value]
-
- if isinstance(value, dict):
- return _parse_nuxt_obj(nuxt_data, value)
-
- return value
-
- return {name: hydrate(nuxt_data[idx]) for name, idx in index_map.items()}
-
+ ld_json = ld_json[0]
-def unescape(html: str) -> str:
- return make_soup(html).get_text()
+ return cast("JsonLD", ld_json)
-def make_soup(html: str) -> BeautifulSoup:
- return BeautifulSoup(html, "html.parser")
+unescape = html.unescape
iframes = CssAttributeSelector("iframe", "src")
diff --git a/cyberdrop_dl/utils/nuxt.py b/cyberdrop_dl/utils/nuxt.py
new file mode 100644
index 000000000..4377e220f
--- /dev/null
+++ b/cyberdrop_dl/utils/nuxt.py
@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import json
+import logging
+from typing import TYPE_CHECKING, Any
+
+from cyberdrop_dl.utils import css
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
+
+ from bs4.element import Tag
+
+
+logger = logging.getLogger(__name__)
+
+
+def extract(soup: Tag) -> list[Any]:
+ return json.loads(css.select_text(soup, "script#__NUXT_DATA__"))
+
+
+def parse_obj(nuxt_data: list[Any], attribute: str, *attributes: str) -> dict[str, Any]:
+ """Parses a single object from a NUXT rich JSON payload response (__NUXT_DATA__)
+
+ It iterates over each object until it finds an object with the desired attributes"""
+ if obj := next(parse_objs(nuxt_data, attribute, *attributes), None):
+ return obj
+ raise css.SelectorError(f"Unable to find object with {attributes = } in NUXT_DATA")
+
+
+def parse_objs(nuxt_data: list[Any], attribute: str, *attributes: str) -> Generator[dict[str, Any]]:
+ """
+ Iterates over each object from a NUXT rich JSON payload response (__NUXT_DATA__)
+
+ It bypasses the devalue parsing logic by ignoring objects without the desired attributes
+
+ https://github.com/nuxt/nuxt/discussions/20879
+ """
+ attributes = attribute, *attributes
+ objects = (obj for obj in nuxt_data if isinstance(obj, dict) and all(key in obj for key in attributes))
+ for obj in objects:
+ try:
+ index: int = obj[attribute]
+ index_map: dict[str, int] = nuxt_data[index]
+ if not isinstance(index_map, dict):
+ raise LookupError
+
+ except LookupError:
+ index_map = obj
+
+ yield _parse_obj(nuxt_data, index_map)
+
+
+def _parse_obj(nuxt_data: list[Any], index_map: dict[str, int]) -> dict[str, Any]:
+ def hydrate(value: Any) -> Any:
+ if isinstance(value, list):
+ match value:
+ case ["BigInt", val]:
+ return int(val)
+ case ["Date" | "Object" | "RegExp", val, *_]:
+ return val
+ case ["Set", *values]:
+ return [hydrate(nuxt_data[idx]) for idx in values]
+ case ["Map", *values]:
+ return hydrate(dict(zip(*(iter(values),) * 2, strict=True)))
+ case ["ShallowRef" | "ShallowReactive" | "Ref" | "Reactive" | "NuxtError", idx]:
+ return hydrate(nuxt_data[idx])
+ case [str(name), *rest]:
+ logger.warning(f"Unable to parse custom object {name} {rest}")
+ return None
+ case _:
+ return [hydrate(nuxt_data[idx]) for idx in value]
+
+ if isinstance(value, dict):
+ return _parse_obj(nuxt_data, value)
+
+ return value
+
+ return {name: hydrate(nuxt_data[idx]) for name, idx in index_map.items()}
diff --git a/tests/test_css.py b/tests/test_css.py
new file mode 100644
index 000000000..1ba0a94e9
--- /dev/null
+++ b/tests/test_css.py
@@ -0,0 +1,34 @@
+import pytest
+
+from cyberdrop_dl.utils import css
+
+
+ "raw, domain, expected",
+ [
+ ("cyberdrop-dl | GitHub", "github.com", "cyberdrop-dl"),
+ ("cyberdrop-dl - GitHub", "github.com", "cyberdrop-dl"),
+ ("cyberdrop-dl | GitHub - Bar", "github.com", "cyberdrop-dl"),
+ ("", "github.com", ""),
+ (" ", "github.com", " "),
+ # case-insensitive
+ ("cyberdrop-dl | GITHUB", "GitHub.com", "cyberdrop-dl"),
+ ("cyberdrop-dl - GiThUb", "GITHUB.io", "cyberdrop-dl"),
+ # sub-domains
+ ("News | www.github.co", "www.github.co.uk", "News"),
+ # no match -> unchanged
+ ("cyberdrop-dl | Foo", "github.com", "cyberdrop-dl | Foo"),
+ ("cyberdrop-dl - Foo", "github.com", "cyberdrop-dl - Foo"),
+ # clean up once
+ ("cyberdrop-dl | Foo - GitHub", "github.com", "cyberdrop-dl | Foo"),
+ ("A | B | GitHub", "github.com", "A | B"),
+ ("A - B - GitHub", "github.com", "A - B"),
+ ],
+)
+def test_rstrip_domain(raw: str, domain: str, expected: str) -> None:
+ assert css.rstrip_domain(raw, domain) == expected
+
+
+def test_no_domain_raise_error() -> None:
+ with pytest.raises(AssertionError):
+ css.rstrip_domain("cyberdrop-dl | Foo", "")
diff --git a/tests/test_nextjs.py b/tests/test_nextjs.py
index 916dece9b..cfb5728aa 100644
--- a/tests/test_nextjs.py
+++ b/tests/test_nextjs.py
@@ -4,14 +4,14 @@ import aiohttp
import pytest
from bs4 import BeautifulSoup
-from cyberdrop_dl.utils import css, next_js
+from cyberdrop_dl.utils import next_js
@pytest.fixture(name="soup", scope="module")
async def onepace_soup() -> AsyncGenerator[BeautifulSoup]:
async with aiohttp.ClientSession() as session:
resp = await session.get("https://onepace.net/en/watch")
- yield css.make_soup(await resp.text())
+ yield BeautifulSoup(await resp.text(), "html.parser")
def test_next_js_parser(soup: BeautifulSoup) -> None: