Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: make all crawlers type safe (#1049)

NTFSvolumeea93a0184b1f
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.py
index b3bd89d83..5d6435d49 100644
--- a/cyberdrop_dl/crawlers/ashemaletube.py
+++ b/cyberdrop_dl/crawlers/ashemaletube.py
@@ -10,6 +10,7 @@ from aiolimiter import AsyncLimiter
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
@@ -158,9 +159,9 @@ class AShemaleTubeCrawler(Crawler):
@error_handling_wrapper
async def proccess_image(self, scrape_item: ScrapeItem, img_tag: Tag) -> None:
if image := img_tag.select_one("img"):
- link_str: str = image["src"]
+ link_str: str = css.get_attr(image, "src")
else:
- style: str = img_tag.select_one("a")["style"]
+ style: str = css.select_one_get_attr(img_tag, "a", "style")
link_str = get_text_between(style, "url('", "');")
url = self.parse_url(link_str).with_query(None)
filename, ext = self.get_filename_and_ext(url.name)
@@ -191,7 +192,7 @@ class AShemaleTubeCrawler(Crawler):
if "uploadDate" in json_data:
scrape_item.possible_datetime = self.parse_date(json_data["uploadDate"])
- title: str = soup.select_one("title").text.split("- aShemaletube.com")[0].strip()
+ title: str = css.select_one_get_text(soup, "title").split("- aShemaletube.com")[0].strip()
link = self.parse_url(best_format.link_str)
scrape_item.url = canonical_url
diff --git a/cyberdrop_dl/crawlers/bestprettygirl.py b/cyberdrop_dl/crawlers/bestprettygirl.py
index e7c42a5e0..50a312684 100644
--- a/cyberdrop_dl/crawlers/bestprettygirl.py
+++ b/cyberdrop_dl/crawlers/bestprettygirl.py
@@ -7,6 +7,7 @@ from aiolimiter import AsyncLimiter
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -83,8 +84,8 @@ class BestPrettyGirlCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- og_title: str = soup.select_one(_SELECTORS.TITLE)["content"]
- date_str: str = soup.select_one(_SELECTORS.DATE)["content"]
+ og_title: str = css.select_one_get_attr(soup, _SELECTORS.TITLE, "content")
+ date_str: str = css.select_one_get_attr(soup, _SELECTORS.DATE, "content")
title = self.create_title(og_title)
scrape_item.setup_as_album(title)
scrape_item.possible_datetime = self.parse_date(date_str)
diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.py
index 9250c1b21..613753323 100644
--- a/cyberdrop_dl/crawlers/box_dot_com.py
+++ b/cyberdrop_dl/crawlers/box_dot_com.py
@@ -12,6 +12,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.models import AliasModel
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedDomains, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -67,7 +68,7 @@ class BoxDotComCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- js_text: str = soup.select_one(JS_SELECTOR).text
+ js_text: str = css.select_one_get_text(soup, JS_SELECTOR)
_, _, data = js_text.removesuffix(";").partition("=")
if not data:
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.py
index 786a097b9..48c53e508 100644
--- a/cyberdrop_dl/crawlers/bunkrr.py
+++ b/cyberdrop_dl/crawlers/bunkrr.py
@@ -156,7 +156,7 @@ class BunkrrCrawler(Crawler):
async def album(self, scrape_item: ScrapeItem) -> None:
soup: BeautifulSoup = await self.get_soup_lenient(scrape_item.url)
album_id = scrape_item.url.parts[2]
- title = soup.select_one("title").text.rsplit(" | Bunkr")[0].strip()
+ title = css.select_one_get_text(soup, "title").rsplit(" | Bunkr")[0].strip()
title = self.create_title(title, album_id)
scrape_item.setup_as_album(title, album_id=album_id)
results = await self.get_album_results(album_id)
@@ -211,7 +211,7 @@ class BunkrrCrawler(Crawler):
# Try image first to not make any aditional request
if image_container:
- link_str: str = image_container.get("src")
+ link_str: str = css.get_attr(image_container, "src")
link = self.parse_url(link_str)
# Try to get downloadd URL from streaming API. Should work for most files, even none video files
@@ -223,7 +223,7 @@ class BunkrrCrawler(Crawler):
# Fallback for everything else, try to get the download URL. `handle_direct_link` will make the final request to the API
if not link and download_link_container:
- link_str: str = download_link_container.get("href")
+ link_str: str = css.get_attr(download_link_container, "href")
link = self.parse_url(link_str)
# Everything failed, abort
@@ -241,7 +241,7 @@ class BunkrrCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- title: str = soup.select_one("h1").text.strip()
+ title: str = css.select_one_get_text(soup, "h1")
link: URL | None = await self.get_download_url_from_api(scrape_item.url)
if not link:
raise ScrapeError(422)
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 5937e1318..e52a66edf 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -34,6 +34,7 @@ from cyberdrop_dl.utils.utilities import (
P = ParamSpec("P")
R = TypeVar("R")
+
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Generator
@@ -417,7 +418,7 @@ class Crawler(ABC):
*,
results: dict[str, int] | None = None,
**kwargs: Any,
- ) -> Generator[tuple[URL | None, ScrapeItem]]:
+ ) -> Generator[tuple[AbsoluteHttpURL | None, ScrapeItem]]:
"""Generates tuples with an URL from the `src` value of the first image tag (AKA the thumbnail) and a new scrape item from the value of `attribute`
:param results: must be the output of `self.get_album_results`.
diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.py
index 3b427d51a..99a6ba2f8 100644
--- a/cyberdrop_dl/crawlers/cyberdrop.py
+++ b/cyberdrop_dl/crawlers/cyberdrop.py
@@ -8,6 +8,7 @@ from bs4 import BeautifulSoup
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -54,7 +55,7 @@ class CyberdropCrawler(Crawler):
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
try:
- title: str = soup.select_one(_SELECTORS.ALBUM_TITLE).text
+ title: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_TITLE)
title = self.create_title(title, album_id)
scrape_item.setup_as_album(title, album_id=album_id)
except AttributeError:
diff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.py
index d2501fa70..bd91b6de9 100644
--- a/cyberdrop_dl/crawlers/dirtyship.py
+++ b/cyberdrop_dl/crawlers/dirtyship.py
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, ClassVar, NamedTuple
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -66,8 +67,8 @@ class DirtyShipCrawler(Crawler):
if not scrape_item.url.suffix == ".jpg":
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- url = next(
- self.parse_url(a["href"]) for a in soup.select(_SELECTORS.SINGLE_PHOTO) if "full" in a.get_text()
+ url = self.parse_url(
+ next(css.get_attr(a, "href") for a in soup.select(_SELECTORS.SINGLE_PHOTO) if "full" in a.get_text())
)
else:
url = scrape_item.url
@@ -94,7 +95,11 @@ class DirtyShipCrawler(Crawler):
)
for img in thumbnails:
- url = img["src"] if img.get("decoding") == "async" else get_highest_resolution_picture(img["srcset"])
+ url = (
+ css.get_attr(img, "src")
+ if img.get("decoding") == "async"
+ else get_highest_resolution_picture(css.get_attr(img, "srcset"))
+ )
if not url:
raise ScrapeError(404)
url = self.parse_url(url)
@@ -106,7 +111,7 @@ class DirtyShipCrawler(Crawler):
title: str = ""
async for soup in self.web_pager(scrape_item.url):
if not title:
- title: str = soup.select_one("title").text
+ title: str = css.select_one_get_text(soup, "title")
title = title.split("Archives - DirtyShip")[0]
title = self.create_title(title)
scrape_item.setup_as_album(title)
@@ -119,16 +124,16 @@ class DirtyShipCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- title: str = soup.select_one("title").text
+ title: str = css.select_one_get_text(soup, "title")
title = title.split(" - DirtyShip")[0]
videos = soup.select(_SELECTORS.VIDEO)
def get_formats():
for video in videos:
- link_str: str = video["src"]
+ link_str: str = css.get_attr(video, "src")
if link_str.startswith("type="):
continue
- res: str = video["title"]
+ res: str = css.get_attr(video, "title")
link = self.parse_url(link_str)
yield (Format(int(res), link))
@@ -146,7 +151,7 @@ class DirtyShipCrawler(Crawler):
def get_flowplayer_sources(self, soup: BeautifulSoup) -> set[Format]:
flow_player = soup.select_one(_SELECTORS.FLOWPLAYER_VIDEO)
- data_item: str | None = flow_player.get("data-item") if flow_player else None
+ data_item: str | None = css.get_attr_or_none(flow_player, "data-item") if flow_player else None
if not data_item:
return set()
data_item = data_item.replace(r"\/", "/")
diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.py
index eea75e58b..254a7d4b0 100644
--- a/cyberdrop_dl/crawlers/doodstream.py
+++ b/cyberdrop_dl/crawlers/doodstream.py
@@ -10,6 +10,7 @@ from yarl import URL
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedDomains, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
@@ -66,7 +67,7 @@ class DoodStreamCrawler(Crawler):
assert host
del response
- title: str = soup.select_one("title").text
+ title: str = css.select_one_get_text(soup, "title")
title = title.split("- DoodStream")[0].strip()
file_id = get_file_id(soup)
diff --git a/cyberdrop_dl/crawlers/ehentai.py b/cyberdrop_dl/crawlers/ehentai.py
index 9d1aa9c85..0fddc447a 100644
--- a/cyberdrop_dl/crawlers/ehentai.py
+++ b/cyberdrop_dl/crawlers/ehentai.py
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -55,8 +56,8 @@ class EHentaiCrawler(Crawler):
scrape_item.url = scrape_item.url.with_query(None)
async for soup in self.web_pager(scrape_item.url):
if not title:
- title = self.create_title(soup.select_one(_SELECTORS.TITLE).get_text())
- date_str: str = soup.select_one(_SELECTORS.DATE).get_text()
+ title = self.create_title(css.select_one_get_text(soup, _SELECTORS.TITLE))
+ date_str: str = css.select_one_get_text(soup, _SELECTORS.DATE)
gallery_id = scrape_item.url.parts[2]
title = self.create_title(title, gallery_id)
scrape_item.setup_as_album(title, album_id=gallery_id)
@@ -73,7 +74,7 @@ class EHentaiCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- link_str: str = soup.select_one(_SELECTORS.IMAGE).get("src")
+ link_str: str = css.select_one_get_attr(soup, _SELECTORS.IMAGE, "src")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
custom_filename, _ = self.get_filename_and_ext(f"{scrape_item.url.name}{ext}")
diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.py
index 52584c327..285240f7a 100644
--- a/cyberdrop_dl/crawlers/eporner.py
+++ b/cyberdrop_dl/crawlers/eporner.py
@@ -117,7 +117,7 @@ class EpornerCrawler(Crawler):
title: str = ""
async for soup in self.web_pager(scrape_item.url):
if not title and not from_profile:
- title = soup.title.text
+ title = css.select_one_get_text(soup, "title")
title_trash = "Porn Star Videos", "Porn Videos", "Videos -", "EPORNER"
for trash in title_trash:
title = title.rsplit(trash)[0].strip()
@@ -132,7 +132,7 @@ class EpornerCrawler(Crawler):
title: str = ""
async for soup in self.web_pager(scrape_item.url):
if not title:
- title = soup.select_one(_SELECTORS.GALLERY_TITLE).get_text(strip=True)
+ title = css.select_one_get_text(soup, _SELECTORS.GALLERY_TITLE)
title = self.create_title(title)
scrape_item.setup_as_album(title)
@@ -157,7 +157,7 @@ class EpornerCrawler(Crawler):
img = soup.select_one(_SELECTORS.PHOTO)
if not img:
raise ScrapeError(422)
- link_str: str = img["href"]
+ link_str: str = css.get_attr(img, "href")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/erome.py b/cyberdrop_dl/crawlers/erome.py
index 8bb7ac83f..3e0873c8f 100644
--- a/cyberdrop_dl/crawlers/erome.py
+++ b/cyberdrop_dl/crawlers/erome.py
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -57,7 +58,7 @@ class EromeCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- title_portion = soup.select_one("title").text.rsplit(" - Porn")[0].strip()
+ title_portion = css.select_one_get_text(soup, "title").rsplit(" - Porn")[0].strip()
if not title_portion:
title_portion = scrape_item.url.name
diff --git a/cyberdrop_dl/crawlers/fapello.py b/cyberdrop_dl/crawlers/fapello.py
index ff824cabe..c0491f08e 100644
--- a/cyberdrop_dl/crawlers/fapello.py
+++ b/cyberdrop_dl/crawlers/fapello.py
@@ -7,6 +7,7 @@ from aiolimiter import AsyncLimiter
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -47,14 +48,13 @@ class FapelloCrawler(Crawler):
title: str = ""
async for soup in self.web_pager(scrape_item.url):
if not title:
- title = self.create_title(soup.select_one(TITLE_SELECTOR).get_text())
+ title = self.create_title(css.select_one_get_text(soup, TITLE_SELECTOR))
scrape_item.setup_as_album(title)
for post in soup.select(CONTENT_SELECTOR):
- link_str: str = post.get("href")
+ link_str: str = css.get_attr(post, "href")
if "javascript" in link_str:
- video_tag = post.select_one("iframe")
- link_str: str = video_tag.get("src")
+ link_str = css.select_one_get_attr(post, "iframe", "src")
link = self.parse_url(link_str)
new_scrape_item = scrape_item.create_child(link)
diff --git a/cyberdrop_dl/crawlers/fileditch.py b/cyberdrop_dl/crawlers/fileditch.py
index 4cd6e44cc..6c11d6544 100644
--- a/cyberdrop_dl/crawlers/fileditch.py
+++ b/cyberdrop_dl/crawlers/fileditch.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -33,7 +34,7 @@ class FileditchCrawler(Crawler):
async def file(self, scrape_item: ScrapeItem) -> None:
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- link_str: str = soup.select_one(DOWNLOAD_SELECTOR).get("href")
+ link_str: str = css.select_one_get_attr(soup, DOWNLOAD_SELECTOR, "href")
link = self.parse_url(link_str)
if link.path == HOMEPAGE_CATCHALL_FILE:
raise ScrapeError(422)
diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.py
index be7b836ef..519fc6fcc 100644
--- a/cyberdrop_dl/crawlers/google_drive.py
+++ b/cyberdrop_dl/crawlers/google_drive.py
@@ -119,7 +119,7 @@ class GoogleDriveCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, download_url)
- title: str = soup.title.text.strip()
+ title: str = css.select_one_get_text(soup, "title")
children = []
for item in soup.select(ITEM_SELECTOR):
link = item.get("href")
@@ -215,7 +215,7 @@ class GoogleDriveCrawler(Crawler):
def get_docs_url(soup: BeautifulSoup, file_id: str) -> AbsoluteHttpURL | None:
- title: str = soup.title.text
+ title: str = css.select_one_get_text(soup, "title")
if title.endswith(" - Google Docs"):
return AbsoluteHttpURL(f"https://docs.google.com/document/d/{file_id}/export?format=docx")
if title.endswith(" - Google Sheets"):
@@ -225,16 +225,14 @@ def get_docs_url(soup: BeautifulSoup, file_id: str) -> AbsoluteHttpURL | None:
def get_id_from_query(url: AbsoluteHttpURL) -> str | None:
- item_id = url.query.get("id")
- if item_id:
+ if item_id := url.query.get("id"):
if len(item_id) == 1:
return item_id[1]
return item_id
def get_file_id(url: AbsoluteHttpURL) -> str:
- file_id = get_id_from_query(url)
- if file_id:
+ if file_id := get_id_from_query(url):
return file_id
if "d" in url.parts and any(p in url.parts for p in VALID_FILE_URL_PARTS):
diff --git a/cyberdrop_dl/crawlers/hotpic.py b/cyberdrop_dl/crawlers/hotpic.py
index 727dc0bf8..bdab14dce 100644
--- a/cyberdrop_dl/crawlers/hotpic.py
+++ b/cyberdrop_dl/crawlers/hotpic.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedDomains, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, with_suffix_encoded
if TYPE_CHECKING:
@@ -42,7 +43,7 @@ class HotPicCrawler(Crawler):
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
album_id = scrape_item.url.parts[2]
- title = self.create_title(soup.title.text.rsplit(" - ")[0], scrape_item.album_id)
+ title = self.create_title(css.select_one_get_text(soup, "title").rsplit(" - ")[0], scrape_item.album_id)
scrape_item.setup_as_profile(title, album_id=album_id)
for _, link in self.iter_tags(soup, ALBUM_ITEM_SELECTOR):
@@ -59,7 +60,7 @@ class HotPicCrawler(Crawler):
file = soup.select_one(VIDEO_SELECTOR) or soup.select_one(IMAGE_SELECTOR)
if not file:
raise ScrapeError(422)
- link_str: str = file.get("src")
+ link_str: str = css.get_attr(file, "src")
link = self.parse_url(link_str)
await self.handle_direct_link(scrape_item, link)
diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.py
index 0c366e563..4bff02b5d 100644
--- a/cyberdrop_dl/crawlers/imagebam.py
+++ b/cyberdrop_dl/crawlers/imagebam.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -46,7 +47,7 @@ class ImageBamCrawler(Crawler):
@error_handling_wrapper
async def gallery(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> None:
- gallery_name = soup.select_one(GALLERY_TITLE_SELECTOR).get_text()
+ gallery_name = css.select_one_get_text(soup, GALLERY_TITLE_SELECTOR)
gallery_id = scrape_item.url.name
title = self.create_title(gallery_name, gallery_id)
scrape_item.setup_as_album(title, album_id=gallery_id)
@@ -67,12 +68,12 @@ class ImageBamCrawler(Crawler):
gallery_info = soup.select_one(GALLERY_INFO_SELECTOR)
if gallery_info and not scrape_item.album_id:
- gallery_url = self.parse_url(gallery_info.get("href"))
+ gallery_url = self.parse_url(css.get_attr(gallery_info, "href"))
gallery_id = gallery_url.name
scrape_item.album_id = gallery_id
- title: str = image_tag["alt"]
- link = self.parse_url(image_tag["src"])
+ title: str = css.get_attr(image_tag, "alt")
+ link = self.parse_url(css.get_attr(image_tag, "src"))
filename, ext = self.get_filename_and_ext(link.name)
custom_filename, _ = self.get_filename_and_ext(title)
await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.py
index a16ff649c..26c70d4ed 100644
--- a/cyberdrop_dl/crawlers/imgbb.py
+++ b/cyberdrop_dl/crawlers/imgbb.py
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedDomains, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -50,10 +51,10 @@ class ImgBBCrawler(Crawler):
async for soup in self.web_pager(scrape_item.url / "sub"):
if not title:
album_id = scrape_item.url.parts[2]
- title_portion = soup.select_one(ALBUM_TITLE_SELECTOR).get_text()
+ title_portion = css.select_one_get_text(soup, ALBUM_TITLE_SELECTOR)
title = self.create_title(title_portion, album_id)
scrape_item.setup_as_album(title, album_id=album_id)
- first_page_str: str = soup.select_one(FIRST_PAGE_SELECTOR).get("href")
+ first_page_str: str = css.select_one_get_attr(soup, FIRST_PAGE_SELECTOR, "href")
first_page = self.parse_url(first_page_str)
for _, sub_album in self.iter_children(scrape_item, soup, ALBUM_PAGE_SELECTOR):
@@ -71,9 +72,9 @@ class ImgBBCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- link_str: str = soup.select_one(IMAGE_SELECTOR).get("src")
+ link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")
link = self.parse_url(link_str)
- date_str: str = soup.select_one(DATE_SELECTOR).get("title")
+ date_str: str = css.select_one_get_attr(soup, DATE_SELECTOR, "title")
scrape_item.possible_datetime = self.parse_date(date_str, "%Y-%m-%d %H:%M:%S")
filename, ext = self.get_filename_and_ext(link.name)
diff --git a/cyberdrop_dl/crawlers/imgbox.py b/cyberdrop_dl/crawlers/imgbox.py
index 38f042145..1756d2a5f 100644
--- a/cyberdrop_dl/crawlers/imgbox.py
+++ b/cyberdrop_dl/crawlers/imgbox.py
@@ -66,7 +66,7 @@ class ImgBoxCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- link_str: str = soup.select_one(IMAGE_SELECTOR).get("src")
+ link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.py
index b6c8a9e75..bb6336516 100644
--- a/cyberdrop_dl/crawlers/imx_to.py
+++ b/cyberdrop_dl/crawlers/imx_to.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -42,7 +43,7 @@ class ImxToCrawler(Crawler):
async with self.request_limiter:
soup = await self.client.post_data_get_soup(self.DOMAIN, scrape_item.url, data=data)
- link_str: str = soup.select_one(IMG_SELECTOR)["src"]
+ link_str: str = css.select_one_get_attr(soup, IMG_SELECTOR, "src")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name, assume_ext=".jpg")
await self.handle_file(link, scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/influencer_bitches.py b/cyberdrop_dl/crawlers/influencer_bitches.py
index eb1d205c9..55a2d44f5 100644
--- a/cyberdrop_dl/crawlers/influencer_bitches.py
+++ b/cyberdrop_dl/crawlers/influencer_bitches.py
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -54,11 +55,11 @@ class InfluencerBitchesCrawler(Crawler):
scrape_item.setup_as_album("Photos", album_id=album_id)
results = await self.get_album_results(album_id)
for a_tag in soup.select(_SELECTORS.PICTURES):
- link_str: str = a_tag.select_one("img")["data-full"]
+ link_str: str = css.select_one_get_attr(a_tag, "img", "data-full")
link = self.parse_url(link_str)
if self.check_album_results(link, results):
continue
- web_url = self.parse_url(a_tag["href"])
+ web_url = self.parse_url(css.get_attr(a_tag, "href"))
new_scrape_item = scrape_item.create_child(web_url)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, new_scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 1bbb13bb8..5f87b0b18 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -9,6 +9,7 @@ from mediafire import MediaFireApi, api
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import MediaFireError, ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -77,7 +78,7 @@ class MediaFireCrawler(Crawler):
raise ScrapeError(422)
scrape_item.possible_datetime = self.parse_date(soup.select(DATE_SELECTOR)[-1].get_text(), "%Y-%m-%d %H:%M:%S")
- link_str: str = link_tag.get("href")
+ link_str: str = css.get_attr(link_tag, "href")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.py
index 8105a90a5..39c190c8c 100644
--- a/cyberdrop_dl/crawlers/mixdrop.py
+++ b/cyberdrop_dl/crawlers/mixdrop.py
@@ -67,7 +67,7 @@ class MixDropCrawler(Crawler):
@staticmethod
def create_download_link(soup: BeautifulSoup) -> AbsoluteHttpURL:
# Defined as a method to simplify subclasses calls
- js_text = soup.select_one(_SELECTOR.JS).text
+ js_text = css.select_one_get_text(soup, _SELECTOR.JS)
file_id = get_text_between(js_text, "|v2||", "|")
parts = get_text_between(js_text, "MDCore||", "|thumbs").split("|")
secure_key = get_text_between(js_text, f"{file_id}|", "|")
diff --git a/cyberdrop_dl/crawlers/noodle_magazine.py b/cyberdrop_dl/crawlers/noodle_magazine.py
index 672a2a0ad..d8a1ef6fd 100644
--- a/cyberdrop_dl/crawlers/noodle_magazine.py
+++ b/cyberdrop_dl/crawlers/noodle_magazine.py
@@ -66,7 +66,7 @@ class NoodleMagazineCrawler(Crawler):
soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, page_url)
if not title:
- search_string: str = soup.select_one(SEARCH_STRING_SELECTOR).text.strip()
+ search_string: str = css.select_one_get_text(soup, SEARCH_STRING_SELECTOR)
title = search_string.rsplit(" videos", 1)[0]
title = self.create_title(f"{title} [search]")
scrape_item.setup_as_album(title)
diff --git a/cyberdrop_dl/crawlers/omegascans.py b/cyberdrop_dl/crawlers/omegascans.py
index 646b1caa4..507eb5f97 100644
--- a/cyberdrop_dl/crawlers/omegascans.py
+++ b/cyberdrop_dl/crawlers/omegascans.py
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -77,7 +78,7 @@ class OmegaScansCrawler(Crawler):
raise ScrapeError(401, "This chapter is premium")
scrape_item.part_of_album = True
- title_parts = soup.select_one("title").get_text().split(" - ")
+ title_parts = css.select_one_get_text(soup, "title").split(" - ")
series_name, chapter_title = title_parts[:2]
series_title = self.create_title(series_name)
scrape_item.add_to_parent_title(series_title)
@@ -86,8 +87,7 @@ class OmegaScansCrawler(Crawler):
date_str = soup.select(DATE_SELECTOR)[-1].get_text()
date = self.parse_date(date_str)
if not date:
- script = soup.select_one(DATE_JS_SELECTOR)
- date_str = script.get_text().split('created_at\\":\\"')[1].split(".")[0]
+ date_str = css.select_one_get_text(soup, DATE_JS_SELECTOR).split('created_at\\":\\"')[1].split(".")[0]
date = self.parse_date(date_str)
scrape_item.possible_datetime = date
diff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.py
index 66aca0177..b477a255b 100644
--- a/cyberdrop_dl/crawlers/pixhost.py
+++ b/cyberdrop_dl/crawlers/pixhost.py
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -42,7 +43,7 @@ class PixHostCrawler(Crawler):
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
album_id = scrape_item.url.name
- title = soup.select_one(GALLERY_TITLE_SELECTOR).text
+ title = css.select_one_get_text(soup, GALLERY_TITLE_SELECTOR)
title = self.create_title(title, album_id)
scrape_item.setup_as_album(title, album_id=album_id)
results = await self.get_album_results(album_id)
@@ -65,7 +66,7 @@ class PixHostCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- link_str: str = soup.select_one(IMAGE_SELECTOR).get("src")
+ link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")
link = self.parse_url(link_str)
await self.handle_direct_link(scrape_item, link)
diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.py
index 44be12f9a..02a4ec6d4 100644
--- a/cyberdrop_dl/crawlers/pkmncards.py
+++ b/cyberdrop_dl/crawlers/pkmncards.py
@@ -139,8 +139,9 @@ class PkmncardsCrawler(Crawler):
page_url = url.with_query(sort="date", ord="auto", display="images")
async for soup in self.web_pager(page_url):
for thumb in soup.select(_SELECTORS.CARD):
- parts: tuple[str, str, str] = thumb.select_one("img")["src"], thumb["href"], thumb["title"]
- link_str, card_page_url_str, title = parts
+ link_str = css.select_one_get_attr(thumb, "img", "src")
+ card_page_url_str = css.get_attr(thumb, "href")
+ title = css.get_attr(thumb, "title")
card_page_url = self.parse_url(card_page_url_str)
download_url = self.parse_url(link_str)
simple_card = create_simple_card(title, download_url)
@@ -153,9 +154,9 @@ class PkmncardsCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- name = soup.select_one(_SELECTORS.CARD_NAME).text
- number = soup.select_one(_SELECTORS.CARD_NUMBER).text
- link_str: str = soup.select_one(_SELECTORS.CARD_DOWNLOAD)["href"]
+ name = css.select_one_get_text(soup, _SELECTORS.CARD_NAME)
+ number = css.select_one_get_text(soup, _SELECTORS.CARD_NUMBER)
+ link_str: str = css.select_one_get_attr(soup, _SELECTORS.CARD_DOWNLOAD, "href")
link = self.parse_url(link_str)
card_set = create_set(soup)
card = Card(name, number, card_set, link)
@@ -173,9 +174,9 @@ class PkmncardsCrawler(Crawler):
custom_filename, _ = self.get_filename_and_ext(f"{card.full_name}{ext}")
await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
- async def handle_simple_card(self, scrape_item: ScrapeItem, simple_card: SimpleCard):
+ async def handle_simple_card(self, scrape_item: ScrapeItem, simple_card: SimpleCard) -> None:
@error_handling_wrapper
- async def get_card_set(self, scrape_item: ScrapeItem):
+ async def get_card_set(self, scrape_item: ScrapeItem) -> CardSet:
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
return create_set(soup)
diff --git a/cyberdrop_dl/crawlers/pornpics.py b/cyberdrop_dl/crawlers/pornpics.py
index 73dc31873..6e2d527a3 100644
--- a/cyberdrop_dl/crawlers/pornpics.py
+++ b/cyberdrop_dl/crawlers/pornpics.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -57,7 +58,7 @@ class PornPicsCrawler(Crawler):
def update_scrape_item(soup: BeautifulSoup) -> None:
selector = "h2" if collection_type == "channel" else "h1"
- title = soup.select_one(selector).text.removesuffix(" Nude Pics").removesuffix(" Porn Pics")
+ title = css.select_one_get_text(soup, selector).removesuffix(" Nude Pics").removesuffix(" Porn Pics")
title = self.create_title(f"{title} [{collection_type}]")
scrape_item.setup_as_profile(title)
@@ -80,7 +81,7 @@ class PornPicsCrawler(Crawler):
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
scrape_item.url = PRIMARY_URL / "galleries" / gallery_id # canonical URL
- title = soup.select_one("h1").text
+ title = css.select_one_get_text(soup, "h1")
title = self.create_title(title, gallery_id)
scrape_item.setup_as_album(title, album_id=gallery_id)
@@ -109,7 +110,7 @@ class PornPicsCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, current_page)
items = soup.select(IMAGE_SELECTOR)
- return soup, tuple(self.parse_url(image.get("href")) for image in items)
+ return soup, tuple(self.parse_url(css.get_attr(image, "href")) for image in items)
async with self.request_limiter:
# The response is JSON but the "content-type" is wrong so we have to request it as text
diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.py
index 3b461bce0..808d195ef 100644
--- a/cyberdrop_dl/crawlers/porntrex.py
+++ b/cyberdrop_dl/crawlers/porntrex.py
@@ -10,6 +10,7 @@ from yarl import URL
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
@@ -88,7 +89,7 @@ class PorntrexCrawler(Crawler):
if "This album is a private album" in soup.text:
raise ScrapeError(401, "Private album")
- title = soup.select_one(_SELECTORS.ALBUM_TITLE).text
+ title = css.select_one_get_text(soup, _SELECTORS.ALBUM_TITLE)
title = self.create_title(title, album_id)
scrape_item.setup_as_album(title)
@@ -123,13 +124,13 @@ class PorntrexCrawler(Crawler):
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
if "models" in scrape_item.url.parts:
- title: str = soup.select_one(_SELECTORS.MODEL_NAME).get_text(strip=True).title()
+ title: str = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME).title()
elif "members" in scrape_item.url.parts:
- title: str = soup.select_one(_SELECTORS.USER_NAME).get_text(strip=True)
+ title: str = css.select_one_get_text(soup, _SELECTORS.USER_NAME)
elif "latest-updates" in scrape_item.url.parts:
title: str = "Latest Updates"
else:
- title = soup.select_one(_SELECTORS.TITLE).get_text(strip=True)
+ title = css.select_one_get_text(soup, _SELECTORS.TITLE)
for trash in TITLE_TRASH:
title = title.replace(trash, "").strip()
@@ -171,6 +172,7 @@ class PorntrexCrawler(Crawler):
sort_by: str = scrape_item.url.parts[4] if len(scrape_item.url.parts) > 4 else ""
sort_by = sort_by or scrape_item.url.query.get("sort_by") or "post_date"
if "search" in scrape_item.url.parts:
+ assert len(scrape_item.url.parts) > 3
search_query = scrape_item.url.parts[3]
block_id = "list_videos_videos"
elif "members" in scrape_item.url.parts:
diff --git a/cyberdrop_dl/crawlers/postimg.py b/cyberdrop_dl/crawlers/postimg.py
index b99bb4280..44895c72f 100644
--- a/cyberdrop_dl/crawlers/postimg.py
+++ b/cyberdrop_dl/crawlers/postimg.py
@@ -7,6 +7,7 @@ from yarl import URL
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -64,7 +65,7 @@ class PostImgCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- link_str: str = soup.select_one(DOWNLOAD_BUTTON_SELECTOR).get("href")
+ link_str: str = css.select_one_get_attr(soup, DOWNLOAD_BUTTON_SELECTOR, "href")
link = self.parse_url(link_str).with_query(None)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.py
index 549744df3..e917d07ba 100644
--- a/cyberdrop_dl/crawlers/redgifs.py
+++ b/cyberdrop_dl/crawlers/redgifs.py
@@ -78,7 +78,7 @@ class RedGifsCrawler(Crawler):
scrape_item.possible_datetime = json_resp["gif"]["createDate"]
links: dict[str, str] = json_resp["gif"]["urls"]
- link_str: str = links.get("hd") or links.get("sd")
+ link_str: str = links.get("hd") or links["sd"]
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/rule34vault.py b/cyberdrop_dl/crawlers/rule34vault.py
index 8cbbb45f2..2ced895d2 100644
--- a/cyberdrop_dl/crawlers/rule34vault.py
+++ b/cyberdrop_dl/crawlers/rule34vault.py
@@ -86,7 +86,8 @@ class Rule34VaultCrawler(Crawler):
scrape_item.url = canonical_url
media_tag = soup.select_one(_SELECTORS.VIDEO) or soup.select_one(_SELECTORS.IMAGE)
- link_str: str = media_tag["src"]
+ assert media_tag
+ link_str: str = css.get_attr(media_tag, "src")
for trash in (".small", ".thumbnail", ".picsmall", ".720", ".hevc"):
link_str = link_str.replace(trash, "")
diff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.py
index ac43e206f..1761bd55a 100644
--- a/cyberdrop_dl/crawlers/rule34video.py
+++ b/cyberdrop_dl/crawlers/rule34video.py
@@ -115,7 +115,7 @@ class Rule34VideoCrawler(Crawler):
def get_video_info(soup: BeautifulSoup) -> VideoInfo:
- title = soup.select_one(VIDEO_TITLE_SELECTOR).text.strip()
+ title = css.select_one_get_text(soup, VIDEO_TITLE_SELECTOR)
info_js_script = soup.select_one(JS_SELECTOR)
info: dict[str, str | dict] = javascript.parse_json_to_dict(info_js_script.text)
info["title"] = title
@@ -126,7 +126,7 @@ def get_video_info(soup: BeautifulSoup) -> VideoInfo:
def get_available_formats(soup: BeautifulSoup) -> Generator[Format]:
for download in soup.select(DOWNLOADS_SELECTOR):
- link_str: str = download.get("href")
+ link_str: str = css.get_attr(download, "href")
if "/tags/" in link_str or not all(p in link_str for p in REQUIRED_FORMAT_STRINGS):
continue
ext, res = download.text.rsplit(" ", 1)
@@ -150,10 +150,10 @@ def get_playlist_title(soup: BeautifulSoup, playlist_type: str) -> str:
selector = PLAYLIST_TITLE_SELECTORS[playlist_type]
title_tag = css.select_one(soup, selector)
if playlist_type in ("tags", "search"):
- for span in title_tag.find_all("span"):
+ for span in title_tag.select("span"):
span.decompose()
- title = css.get_text(title_tag.text)
+ title = css.get_text(title_tag)
for trash in TITLE_TRASH:
title = title.replace(trash, "").strip()
diff --git a/cyberdrop_dl/crawlers/rule34xxx.py b/cyberdrop_dl/crawlers/rule34xxx.py
index b643f91c4..970edfb73 100644
--- a/cyberdrop_dl/crawlers/rule34xxx.py
+++ b/cyberdrop_dl/crawlers/rule34xxx.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -66,7 +67,7 @@ class Rule34XXXCrawler(Crawler):
if date_tag := soup.select_one(_SELECTOR.DATE):
date_str = date_tag.get_text(strip=True).removeprefix("Posted: ")
scrape_item.possible_datetime = self.parse_date(date_str)
- link_str: str = media_tag["src"]
+ link_str: str = css.get_attr(media_tag, "src")
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
diff --git a/cyberdrop_dl/crawlers/thisvid.py b/cyberdrop_dl/crawlers/thisvid.py
index ee3613106..e4da4230b 100644
--- a/cyberdrop_dl/crawlers/thisvid.py
+++ b/cyberdrop_dl/crawlers/thisvid.py
@@ -158,7 +158,7 @@ class ThisVidCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- js_text: str = soup.select_one(_SELECTORS.ALBUM_ID).text
+ js_text: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_ID)
album_id: str = get_text_between(js_text, "params['album_id'] =", ";").strip()
results = await self.get_album_results(album_id)
title: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_NAME)
diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.py
index 8d75e4303..398b2f78e 100644
--- a/cyberdrop_dl/crawlers/tokyomotion.py
+++ b/cyberdrop_dl/crawlers/tokyomotion.py
@@ -1,12 +1,12 @@
from __future__ import annotations
-import contextlib
import re
from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
+from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts
if TYPE_CHECKING:
@@ -76,22 +76,17 @@ class TokioMotionCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- with contextlib.suppress(AttributeError):
- relative_date_str = soup.select_one(self.video_date_selector).text.strip()
- scrape_item.possible_datetime = self.parse_date(relative_date_str)
+ scrape_item.possible_datetime = self.parse_date(css.select_one_get_text(soup, self.video_date_selector))
- try:
- srcSD = soup.select_one('source[title="SD"]')
- srcHD = soup.select_one('source[title="HD"]')
- src = srcHD or srcSD
- link_str: str = src.get("src")
- link = self.parse_url(link_str)
- except AttributeError:
+ src = soup.select_one('source[title="SD"]') or soup.select_one('source[title="HD"]')
+ if not src:
if "This is a private" in soup.text:
- raise ScrapeError(401, "Private video") from None
- raise ScrapeError(422, "Couldn't find video source") from None
+ raise ScrapeError(401, "Private video")
+ raise ScrapeError(422, "Couldn't find video source")
- title = soup.select_one("title").text.rsplit(" - TOKYO Motion")[0].strip()
+ link_str: str = css.get_attr(src, "src")
+ link = self.parse_url(link_str)
+ title = css.select_one_get_text(soup, "title").rsplit(" - TOKYO Motion")[0].strip()
filename, ext = f"{video_id}.mp4", ".mp4"
custom_filename, _ = self.get_filename_and_ext(f"{title} [{video_id}]{ext}")
await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
@@ -103,15 +98,15 @@ class TokioMotionCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- try:
- img = soup.select_one(self.image_selector)
- link_str: str = img.get("src")
- link = self.parse_url(link_str)
- except AttributeError:
+
+ img = soup.select_one(self.image_selector)
+ if not img:
if "This is a private" in soup.text:
- raise ScrapeError(401, "Private Photo") from None
- raise ScrapeError(422, "Couldn't find image source") from None
+ raise ScrapeError(401, "Private Photo")
+ raise ScrapeError(422, "Couldn't find image source")
+ link_str: str = css.get_attr(img, "src")
+ link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
@@ -206,7 +201,7 @@ class TokioMotionCrawler(Crawler):
return scrape_item.url.parts[3]
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- return soup.select_one(self.album_title_selector).get_text()
+ return css.select_one_get_text(soup, self.album_title_selector)
def add_user_title(self, scrape_item: ScrapeItem) -> None:
try:
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index 26488e395..71422830a 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -79,7 +79,7 @@ from cyberdrop_dl.types import AnyDict, U32Int, U32IntArray, U32IntSequence
from cyberdrop_dl.utils.logger import log
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Generator
+ from collections.abc import AsyncGenerator, Generator, Mapping
from yarl import URL
@@ -742,7 +742,7 @@ class MegaApi:
nodes = {node["h"]: node async for node in prepare_nodes()}
return nodes
- async def _build_file_system(self, nodes_map: dict[str, Node], root_ids: list[str]) -> dict[Path, Node]:
+ async def _build_file_system(self, nodes_map: Mapping[str, Node], root_ids: list[str]) -> dict[Path, Node]:
"""Builds a flattened dictionary representing a file system from a list of items.
Returns:
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 845f80a86..a9d4f4017 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -28,7 +28,7 @@ def select_one(tag: Tag, selector: str) -> Tag:
def select_one_get_text(tag: Tag, selector: str, strip: bool = True) -> str:
- """Same as `tag.select_one` but asserts the result is not `None`"""
+ """Same as `tag.select_one.get_text(strip=strip)` but asserts the result is not `None`"""
return get_text(select_one(tag, selector), strip)