Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: make all crawlers type safe (#1049)
NTFSvolumeea93a0184b1f
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.pyindex b3bd89d83..5d6435d49 100644--- a/cyberdrop_dl/crawlers/ashemaletube.py+++ b/cyberdrop_dl/crawlers/ashemaletube.py@@ -10,6 +10,7 @@ from aiolimiter import AsyncLimiterfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:@@ -158,9 +159,9 @@ class AShemaleTubeCrawler(Crawler):@error_handling_wrapperasync def proccess_image(self, scrape_item: ScrapeItem, img_tag: Tag) -> None:if image := img_tag.select_one("img"):- link_str: str = image["src"]+ link_str: str = css.get_attr(image, "src")else:- style: str = img_tag.select_one("a")["style"]+ style: str = css.select_one_get_attr(img_tag, "a", "style")link_str = get_text_between(style, "url('", "');")url = self.parse_url(link_str).with_query(None)filename, ext = self.get_filename_and_ext(url.name)@@ -191,7 +192,7 @@ class AShemaleTubeCrawler(Crawler):if "uploadDate" in json_data:scrape_item.possible_datetime = self.parse_date(json_data["uploadDate"])- title: str = soup.select_one("title").text.split("- aShemaletube.com")[0].strip()+ title: str = css.select_one_get_text(soup, "title").split("- aShemaletube.com")[0].strip()link = self.parse_url(best_format.link_str)scrape_item.url = canonical_urldiff --git a/cyberdrop_dl/crawlers/bestprettygirl.py b/cyberdrop_dl/crawlers/bestprettygirl.pyindex e7c42a5e0..50a312684 100644--- a/cyberdrop_dl/crawlers/bestprettygirl.py+++ b/cyberdrop_dl/crawlers/bestprettygirl.py@@ -7,6 +7,7 @@ from aiolimiter import AsyncLimiterfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -83,8 +84,8 @@ class BestPrettyGirlCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- og_title: str = soup.select_one(_SELECTORS.TITLE)["content"]- date_str: str = soup.select_one(_SELECTORS.DATE)["content"]+ og_title: str = css.select_one_get_attr(soup, _SELECTORS.TITLE, "content")+ date_str: str = css.select_one_get_attr(soup, _SELECTORS.DATE, "content")title = self.create_title(og_title)scrape_item.setup_as_album(title)scrape_item.possible_datetime = self.parse_date(date_str)diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.pyindex 9250c1b21..613753323 100644--- a/cyberdrop_dl/crawlers/box_dot_com.py+++ b/cyberdrop_dl/crawlers/box_dot_com.py@@ -12,6 +12,7 @@ from cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.models import AliasModelfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedDomains, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -67,7 +68,7 @@ class BoxDotComCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- js_text: str = soup.select_one(JS_SELECTOR).text+ js_text: str = css.select_one_get_text(soup, JS_SELECTOR)_, _, data = js_text.removesuffix(";").partition("=")if not data:diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.pyindex 786a097b9..48c53e508 100644--- a/cyberdrop_dl/crawlers/bunkrr.py+++ b/cyberdrop_dl/crawlers/bunkrr.py@@ -156,7 +156,7 @@ class BunkrrCrawler(Crawler):async def album(self, scrape_item: ScrapeItem) -> None:soup: BeautifulSoup = await self.get_soup_lenient(scrape_item.url)album_id = scrape_item.url.parts[2]- title = soup.select_one("title").text.rsplit(" | Bunkr")[0].strip()+ title = css.select_one_get_text(soup, "title").rsplit(" | Bunkr")[0].strip()title = self.create_title(title, album_id)scrape_item.setup_as_album(title, album_id=album_id)results = await self.get_album_results(album_id)@@ -211,7 +211,7 @@ class BunkrrCrawler(Crawler):# Try image first to not make any aditional requestif image_container:- link_str: str = image_container.get("src")+ link_str: str = css.get_attr(image_container, "src")link = self.parse_url(link_str)# Try to get downloadd URL from streaming API. Should work for most files, even none video files@@ -223,7 +223,7 @@ class BunkrrCrawler(Crawler):# Fallback for everything else, try to get the download URL. `handle_direct_link` will make the final request to the APIif not link and download_link_container:- link_str: str = download_link_container.get("href")+ link_str: str = css.get_attr(download_link_container, "href")link = self.parse_url(link_str)# Everything failed, abort@@ -241,7 +241,7 @@ class BunkrrCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- title: str = soup.select_one("h1").text.strip()+ title: str = css.select_one_get_text(soup, "h1")link: URL | None = await self.get_download_url_from_api(scrape_item.url)if not link:raise ScrapeError(422)diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 5937e1318..e52a66edf 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -34,6 +34,7 @@ from cyberdrop_dl.utils.utilities import (P = ParamSpec("P")R = TypeVar("R")+if TYPE_CHECKING:from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Generator@@ -417,7 +418,7 @@ class Crawler(ABC):*,results: dict[str, int] | None = None,**kwargs: Any,- ) -> Generator[tuple[URL | None, ScrapeItem]]:+ ) -> Generator[tuple[AbsoluteHttpURL | None, ScrapeItem]]:"""Generates tuples with an URL from the `src` value of the first image tag (AKA the thumbnail) and a new scrape item from the value of `attribute`:param results: must be the output of `self.get_album_results`.diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.pyindex 3b427d51a..99a6ba2f8 100644--- a/cyberdrop_dl/crawlers/cyberdrop.py+++ b/cyberdrop_dl/crawlers/cyberdrop.py@@ -8,6 +8,7 @@ from bs4 import BeautifulSoupfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -54,7 +55,7 @@ class CyberdropCrawler(Crawler):soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)try:- title: str = soup.select_one(_SELECTORS.ALBUM_TITLE).text+ title: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_TITLE)title = self.create_title(title, album_id)scrape_item.setup_as_album(title, album_id=album_id)except AttributeError:diff --git a/cyberdrop_dl/crawlers/dirtyship.py b/cyberdrop_dl/crawlers/dirtyship.pyindex d2501fa70..bd91b6de9 100644--- a/cyberdrop_dl/crawlers/dirtyship.py+++ b/cyberdrop_dl/crawlers/dirtyship.py@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, ClassVar, NamedTuplefrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -66,8 +67,8 @@ class DirtyShipCrawler(Crawler):if not scrape_item.url.suffix == ".jpg":async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- url = next(- self.parse_url(a["href"]) for a in soup.select(_SELECTORS.SINGLE_PHOTO) if "full" in a.get_text()+ url = self.parse_url(+ next(css.get_attr(a, "href") for a in soup.select(_SELECTORS.SINGLE_PHOTO) if "full" in a.get_text()))else:url = scrape_item.url@@ -94,7 +95,11 @@ class DirtyShipCrawler(Crawler):)for img in thumbnails:- url = img["src"] if img.get("decoding") == "async" else get_highest_resolution_picture(img["srcset"])+ url = (+ css.get_attr(img, "src")+ if img.get("decoding") == "async"+ else get_highest_resolution_picture(css.get_attr(img, "srcset"))+ )if not url:raise ScrapeError(404)url = self.parse_url(url)@@ -106,7 +111,7 @@ class DirtyShipCrawler(Crawler):title: str = ""async for soup in self.web_pager(scrape_item.url):if not title:- title: str = soup.select_one("title").text+ title: str = css.select_one_get_text(soup, "title")title = title.split("Archives - DirtyShip")[0]title = self.create_title(title)scrape_item.setup_as_album(title)@@ -119,16 +124,16 @@ class DirtyShipCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- title: str = soup.select_one("title").text+ title: str = css.select_one_get_text(soup, "title")title = title.split(" - DirtyShip")[0]videos = soup.select(_SELECTORS.VIDEO)def get_formats():for video in videos:- link_str: str = video["src"]+ link_str: str = css.get_attr(video, "src")if link_str.startswith("type="):continue- res: str = video["title"]+ res: str = css.get_attr(video, "title")link = self.parse_url(link_str)yield (Format(int(res), link))@@ -146,7 +151,7 @@ class DirtyShipCrawler(Crawler):def get_flowplayer_sources(self, soup: BeautifulSoup) -> set[Format]:flow_player = soup.select_one(_SELECTORS.FLOWPLAYER_VIDEO)- data_item: str | None = flow_player.get("data-item") if flow_player else None+ data_item: str | None = css.get_attr_or_none(flow_player, "data-item") if flow_player else Noneif not data_item:return set()data_item = data_item.replace(r"\/", "/")diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.pyindex eea75e58b..254a7d4b0 100644--- a/cyberdrop_dl/crawlers/doodstream.py+++ b/cyberdrop_dl/crawlers/doodstream.py@@ -10,6 +10,7 @@ from yarl import URLfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedDomains, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:@@ -66,7 +67,7 @@ class DoodStreamCrawler(Crawler):assert hostdel response- title: str = soup.select_one("title").text+ title: str = css.select_one_get_text(soup, "title")title = title.split("- DoodStream")[0].strip()file_id = get_file_id(soup)diff --git a/cyberdrop_dl/crawlers/ehentai.py b/cyberdrop_dl/crawlers/ehentai.pyindex 9d1aa9c85..0fddc447a 100644--- a/cyberdrop_dl/crawlers/ehentai.py+++ b/cyberdrop_dl/crawlers/ehentai.py@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -55,8 +56,8 @@ class EHentaiCrawler(Crawler):scrape_item.url = scrape_item.url.with_query(None)async for soup in self.web_pager(scrape_item.url):if not title:- title = self.create_title(soup.select_one(_SELECTORS.TITLE).get_text())- date_str: str = soup.select_one(_SELECTORS.DATE).get_text()+ title = self.create_title(css.select_one_get_text(soup, _SELECTORS.TITLE))+ date_str: str = css.select_one_get_text(soup, _SELECTORS.DATE)gallery_id = scrape_item.url.parts[2]title = self.create_title(title, gallery_id)scrape_item.setup_as_album(title, album_id=gallery_id)@@ -73,7 +74,7 @@ class EHentaiCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- link_str: str = soup.select_one(_SELECTORS.IMAGE).get("src")+ link_str: str = css.select_one_get_attr(soup, _SELECTORS.IMAGE, "src")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)custom_filename, _ = self.get_filename_and_ext(f"{scrape_item.url.name}{ext}")diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.pyindex 52584c327..285240f7a 100644--- a/cyberdrop_dl/crawlers/eporner.py+++ b/cyberdrop_dl/crawlers/eporner.py@@ -117,7 +117,7 @@ class EpornerCrawler(Crawler):title: str = ""async for soup in self.web_pager(scrape_item.url):if not title and not from_profile:- title = soup.title.text+ title = css.select_one_get_text(soup, "title")title_trash = "Porn Star Videos", "Porn Videos", "Videos -", "EPORNER"for trash in title_trash:title = title.rsplit(trash)[0].strip()@@ -132,7 +132,7 @@ class EpornerCrawler(Crawler):title: str = ""async for soup in self.web_pager(scrape_item.url):if not title:- title = soup.select_one(_SELECTORS.GALLERY_TITLE).get_text(strip=True)+ title = css.select_one_get_text(soup, _SELECTORS.GALLERY_TITLE)title = self.create_title(title)scrape_item.setup_as_album(title)@@ -157,7 +157,7 @@ class EpornerCrawler(Crawler):img = soup.select_one(_SELECTORS.PHOTO)if not img:raise ScrapeError(422)- link_str: str = img["href"]+ link_str: str = css.get_attr(img, "href")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/erome.py b/cyberdrop_dl/crawlers/erome.pyindex 8bb7ac83f..3e0873c8f 100644--- a/cyberdrop_dl/crawlers/erome.py+++ b/cyberdrop_dl/crawlers/erome.py@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -57,7 +58,7 @@ class EromeCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- title_portion = soup.select_one("title").text.rsplit(" - Porn")[0].strip()+ title_portion = css.select_one_get_text(soup, "title").rsplit(" - Porn")[0].strip()if not title_portion:title_portion = scrape_item.url.namediff --git a/cyberdrop_dl/crawlers/fapello.py b/cyberdrop_dl/crawlers/fapello.pyindex ff824cabe..c0491f08e 100644--- a/cyberdrop_dl/crawlers/fapello.py+++ b/cyberdrop_dl/crawlers/fapello.py@@ -7,6 +7,7 @@ from aiolimiter import AsyncLimiterfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -47,14 +48,13 @@ class FapelloCrawler(Crawler):title: str = ""async for soup in self.web_pager(scrape_item.url):if not title:- title = self.create_title(soup.select_one(TITLE_SELECTOR).get_text())+ title = self.create_title(css.select_one_get_text(soup, TITLE_SELECTOR))scrape_item.setup_as_album(title)for post in soup.select(CONTENT_SELECTOR):- link_str: str = post.get("href")+ link_str: str = css.get_attr(post, "href")if "javascript" in link_str:- video_tag = post.select_one("iframe")- link_str: str = video_tag.get("src")+ link_str = css.select_one_get_attr(post, "iframe", "src")link = self.parse_url(link_str)new_scrape_item = scrape_item.create_child(link)diff --git a/cyberdrop_dl/crawlers/fileditch.py b/cyberdrop_dl/crawlers/fileditch.pyindex 4cd6e44cc..6c11d6544 100644--- a/cyberdrop_dl/crawlers/fileditch.py+++ b/cyberdrop_dl/crawlers/fileditch.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -33,7 +34,7 @@ class FileditchCrawler(Crawler):async def file(self, scrape_item: ScrapeItem) -> None:async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- link_str: str = soup.select_one(DOWNLOAD_SELECTOR).get("href")+ link_str: str = css.select_one_get_attr(soup, DOWNLOAD_SELECTOR, "href")link = self.parse_url(link_str)if link.path == HOMEPAGE_CATCHALL_FILE:raise ScrapeError(422)diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.pyindex be7b836ef..519fc6fcc 100644--- a/cyberdrop_dl/crawlers/google_drive.py+++ b/cyberdrop_dl/crawlers/google_drive.py@@ -119,7 +119,7 @@ class GoogleDriveCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, download_url)- title: str = soup.title.text.strip()+ title: str = css.select_one_get_text(soup, "title")children = []for item in soup.select(ITEM_SELECTOR):link = item.get("href")@@ -215,7 +215,7 @@ class GoogleDriveCrawler(Crawler):def get_docs_url(soup: BeautifulSoup, file_id: str) -> AbsoluteHttpURL | None:- title: str = soup.title.text+ title: str = css.select_one_get_text(soup, "title")if title.endswith(" - Google Docs"):return AbsoluteHttpURL(f"https://docs.google.com/document/d/{file_id}/export?format=docx")if title.endswith(" - Google Sheets"):@@ -225,16 +225,14 @@ def get_docs_url(soup: BeautifulSoup, file_id: str) -> AbsoluteHttpURL | None:def get_id_from_query(url: AbsoluteHttpURL) -> str | None:- item_id = url.query.get("id")- if item_id:+ if item_id := url.query.get("id"):if len(item_id) == 1:return item_id[1]return item_iddef get_file_id(url: AbsoluteHttpURL) -> str:- file_id = get_id_from_query(url)- if file_id:+ if file_id := get_id_from_query(url):return file_idif "d" in url.parts and any(p in url.parts for p in VALID_FILE_URL_PARTS):diff --git a/cyberdrop_dl/crawlers/hotpic.py b/cyberdrop_dl/crawlers/hotpic.pyindex 727dc0bf8..bdab14dce 100644--- a/cyberdrop_dl/crawlers/hotpic.py+++ b/cyberdrop_dl/crawlers/hotpic.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedDomains, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, with_suffix_encodedif TYPE_CHECKING:@@ -42,7 +43,7 @@ class HotPicCrawler(Crawler):soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)album_id = scrape_item.url.parts[2]- title = self.create_title(soup.title.text.rsplit(" - ")[0], scrape_item.album_id)+ title = self.create_title(css.select_one_get_text(soup, "title").rsplit(" - ")[0], scrape_item.album_id)scrape_item.setup_as_profile(title, album_id=album_id)for _, link in self.iter_tags(soup, ALBUM_ITEM_SELECTOR):@@ -59,7 +60,7 @@ class HotPicCrawler(Crawler):file = soup.select_one(VIDEO_SELECTOR) or soup.select_one(IMAGE_SELECTOR)if not file:raise ScrapeError(422)- link_str: str = file.get("src")+ link_str: str = css.get_attr(file, "src")link = self.parse_url(link_str)await self.handle_direct_link(scrape_item, link)diff --git a/cyberdrop_dl/crawlers/imagebam.py b/cyberdrop_dl/crawlers/imagebam.pyindex 0c366e563..4bff02b5d 100644--- a/cyberdrop_dl/crawlers/imagebam.py+++ b/cyberdrop_dl/crawlers/imagebam.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -46,7 +47,7 @@ class ImageBamCrawler(Crawler):@error_handling_wrapperasync def gallery(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> None:- gallery_name = soup.select_one(GALLERY_TITLE_SELECTOR).get_text()+ gallery_name = css.select_one_get_text(soup, GALLERY_TITLE_SELECTOR)gallery_id = scrape_item.url.nametitle = self.create_title(gallery_name, gallery_id)scrape_item.setup_as_album(title, album_id=gallery_id)@@ -67,12 +68,12 @@ class ImageBamCrawler(Crawler):gallery_info = soup.select_one(GALLERY_INFO_SELECTOR)if gallery_info and not scrape_item.album_id:- gallery_url = self.parse_url(gallery_info.get("href"))+ gallery_url = self.parse_url(css.get_attr(gallery_info, "href"))gallery_id = gallery_url.namescrape_item.album_id = gallery_id- title: str = image_tag["alt"]- link = self.parse_url(image_tag["src"])+ title: str = css.get_attr(image_tag, "alt")+ link = self.parse_url(css.get_attr(image_tag, "src"))filename, ext = self.get_filename_and_ext(link.name)custom_filename, _ = self.get_filename_and_ext(title)await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.pyindex a16ff649c..26c70d4ed 100644--- a/cyberdrop_dl/crawlers/imgbb.py+++ b/cyberdrop_dl/crawlers/imgbb.py@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedDomains, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -50,10 +51,10 @@ class ImgBBCrawler(Crawler):async for soup in self.web_pager(scrape_item.url / "sub"):if not title:album_id = scrape_item.url.parts[2]- title_portion = soup.select_one(ALBUM_TITLE_SELECTOR).get_text()+ title_portion = css.select_one_get_text(soup, ALBUM_TITLE_SELECTOR)title = self.create_title(title_portion, album_id)scrape_item.setup_as_album(title, album_id=album_id)- first_page_str: str = soup.select_one(FIRST_PAGE_SELECTOR).get("href")+ first_page_str: str = css.select_one_get_attr(soup, FIRST_PAGE_SELECTOR, "href")first_page = self.parse_url(first_page_str)for _, sub_album in self.iter_children(scrape_item, soup, ALBUM_PAGE_SELECTOR):@@ -71,9 +72,9 @@ class ImgBBCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- link_str: str = soup.select_one(IMAGE_SELECTOR).get("src")+ link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")link = self.parse_url(link_str)- date_str: str = soup.select_one(DATE_SELECTOR).get("title")+ date_str: str = css.select_one_get_attr(soup, DATE_SELECTOR, "title")scrape_item.possible_datetime = self.parse_date(date_str, "%Y-%m-%d %H:%M:%S")filename, ext = self.get_filename_and_ext(link.name)diff --git a/cyberdrop_dl/crawlers/imgbox.py b/cyberdrop_dl/crawlers/imgbox.pyindex 38f042145..1756d2a5f 100644--- a/cyberdrop_dl/crawlers/imgbox.py+++ b/cyberdrop_dl/crawlers/imgbox.py@@ -66,7 +66,7 @@ class ImgBoxCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- link_str: str = soup.select_one(IMAGE_SELECTOR).get("src")+ link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/imx_to.py b/cyberdrop_dl/crawlers/imx_to.pyindex b6c8a9e75..bb6336516 100644--- a/cyberdrop_dl/crawlers/imx_to.py+++ b/cyberdrop_dl/crawlers/imx_to.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -42,7 +43,7 @@ class ImxToCrawler(Crawler):async with self.request_limiter:soup = await self.client.post_data_get_soup(self.DOMAIN, scrape_item.url, data=data)- link_str: str = soup.select_one(IMG_SELECTOR)["src"]+ link_str: str = css.select_one_get_attr(soup, IMG_SELECTOR, "src")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name, assume_ext=".jpg")await self.handle_file(link, scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/influencer_bitches.py b/cyberdrop_dl/crawlers/influencer_bitches.pyindex eb1d205c9..55a2d44f5 100644--- a/cyberdrop_dl/crawlers/influencer_bitches.py+++ b/cyberdrop_dl/crawlers/influencer_bitches.py@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -54,11 +55,11 @@ class InfluencerBitchesCrawler(Crawler):scrape_item.setup_as_album("Photos", album_id=album_id)results = await self.get_album_results(album_id)for a_tag in soup.select(_SELECTORS.PICTURES):- link_str: str = a_tag.select_one("img")["data-full"]+ link_str: str = css.select_one_get_attr(a_tag, "img", "data-full")link = self.parse_url(link_str)if self.check_album_results(link, results):continue- web_url = self.parse_url(a_tag["href"])+ web_url = self.parse_url(css.get_attr(a_tag, "href"))new_scrape_item = scrape_item.create_child(web_url)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, new_scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 1bbb13bb8..5f87b0b18 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -9,6 +9,7 @@ from mediafire import MediaFireApi, apifrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import MediaFireError, ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -77,7 +78,7 @@ class MediaFireCrawler(Crawler):raise ScrapeError(422)scrape_item.possible_datetime = self.parse_date(soup.select(DATE_SELECTOR)[-1].get_text(), "%Y-%m-%d %H:%M:%S")- link_str: str = link_tag.get("href")+ link_str: str = css.get_attr(link_tag, "href")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.pyindex 8105a90a5..39c190c8c 100644--- a/cyberdrop_dl/crawlers/mixdrop.py+++ b/cyberdrop_dl/crawlers/mixdrop.py@@ -67,7 +67,7 @@ class MixDropCrawler(Crawler):@staticmethoddef create_download_link(soup: BeautifulSoup) -> AbsoluteHttpURL:# Defined as a method to simplify subclasses calls- js_text = soup.select_one(_SELECTOR.JS).text+ js_text = css.select_one_get_text(soup, _SELECTOR.JS)file_id = get_text_between(js_text, "|v2||", "|")parts = get_text_between(js_text, "MDCore||", "|thumbs").split("|")secure_key = get_text_between(js_text, f"{file_id}|", "|")diff --git a/cyberdrop_dl/crawlers/noodle_magazine.py b/cyberdrop_dl/crawlers/noodle_magazine.pyindex 672a2a0ad..d8a1ef6fd 100644--- a/cyberdrop_dl/crawlers/noodle_magazine.py+++ b/cyberdrop_dl/crawlers/noodle_magazine.py@@ -66,7 +66,7 @@ class NoodleMagazineCrawler(Crawler):soup: BeautifulSoup = await self.client.get_soup_cffi(self.DOMAIN, page_url)if not title:- search_string: str = soup.select_one(SEARCH_STRING_SELECTOR).text.strip()+ search_string: str = css.select_one_get_text(soup, SEARCH_STRING_SELECTOR)title = search_string.rsplit(" videos", 1)[0]title = self.create_title(f"{title} [search]")scrape_item.setup_as_album(title)diff --git a/cyberdrop_dl/crawlers/omegascans.py b/cyberdrop_dl/crawlers/omegascans.pyindex 646b1caa4..507eb5f97 100644--- a/cyberdrop_dl/crawlers/omegascans.py+++ b/cyberdrop_dl/crawlers/omegascans.py@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -77,7 +78,7 @@ class OmegaScansCrawler(Crawler):raise ScrapeError(401, "This chapter is premium")scrape_item.part_of_album = True- title_parts = soup.select_one("title").get_text().split(" - ")+ title_parts = css.select_one_get_text(soup, "title").split(" - ")series_name, chapter_title = title_parts[:2]series_title = self.create_title(series_name)scrape_item.add_to_parent_title(series_title)@@ -86,8 +87,7 @@ class OmegaScansCrawler(Crawler):date_str = soup.select(DATE_SELECTOR)[-1].get_text()date = self.parse_date(date_str)if not date:- script = soup.select_one(DATE_JS_SELECTOR)- date_str = script.get_text().split('created_at\\":\\"')[1].split(".")[0]+ date_str = css.select_one_get_text(soup, DATE_JS_SELECTOR).split('created_at\\":\\"')[1].split(".")[0]date = self.parse_date(date_str)scrape_item.possible_datetime = datediff --git a/cyberdrop_dl/crawlers/pixhost.py b/cyberdrop_dl/crawlers/pixhost.pyindex 66aca0177..b477a255b 100644--- a/cyberdrop_dl/crawlers/pixhost.py+++ b/cyberdrop_dl/crawlers/pixhost.py@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -42,7 +43,7 @@ class PixHostCrawler(Crawler):soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)album_id = scrape_item.url.name- title = soup.select_one(GALLERY_TITLE_SELECTOR).text+ title = css.select_one_get_text(soup, GALLERY_TITLE_SELECTOR)title = self.create_title(title, album_id)scrape_item.setup_as_album(title, album_id=album_id)results = await self.get_album_results(album_id)@@ -65,7 +66,7 @@ class PixHostCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- link_str: str = soup.select_one(IMAGE_SELECTOR).get("src")+ link_str: str = css.select_one_get_attr(soup, IMAGE_SELECTOR, "src")link = self.parse_url(link_str)await self.handle_direct_link(scrape_item, link)diff --git a/cyberdrop_dl/crawlers/pkmncards.py b/cyberdrop_dl/crawlers/pkmncards.pyindex 44be12f9a..02a4ec6d4 100644--- a/cyberdrop_dl/crawlers/pkmncards.py+++ b/cyberdrop_dl/crawlers/pkmncards.py@@ -139,8 +139,9 @@ class PkmncardsCrawler(Crawler):page_url = url.with_query(sort="date", ord="auto", display="images")async for soup in self.web_pager(page_url):for thumb in soup.select(_SELECTORS.CARD):- parts: tuple[str, str, str] = thumb.select_one("img")["src"], thumb["href"], thumb["title"]- link_str, card_page_url_str, title = parts+ link_str = css.select_one_get_attr(thumb, "img", "src")+ card_page_url_str = css.get_attr(thumb, "href")+ title = css.get_attr(thumb, "title")card_page_url = self.parse_url(card_page_url_str)download_url = self.parse_url(link_str)simple_card = create_simple_card(title, download_url)@@ -153,9 +154,9 @@ class PkmncardsCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- name = soup.select_one(_SELECTORS.CARD_NAME).text- number = soup.select_one(_SELECTORS.CARD_NUMBER).text- link_str: str = soup.select_one(_SELECTORS.CARD_DOWNLOAD)["href"]+ name = css.select_one_get_text(soup, _SELECTORS.CARD_NAME)+ number = css.select_one_get_text(soup, _SELECTORS.CARD_NUMBER)+ link_str: str = css.select_one_get_attr(soup, _SELECTORS.CARD_DOWNLOAD, "href")link = self.parse_url(link_str)card_set = create_set(soup)card = Card(name, number, card_set, link)@@ -173,9 +174,9 @@ class PkmncardsCrawler(Crawler):custom_filename, _ = self.get_filename_and_ext(f"{card.full_name}{ext}")await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)- async def handle_simple_card(self, scrape_item: ScrapeItem, simple_card: SimpleCard):+ async def handle_simple_card(self, scrape_item: ScrapeItem, simple_card: SimpleCard) -> None:@error_handling_wrapper- async def get_card_set(self, scrape_item: ScrapeItem):+ async def get_card_set(self, scrape_item: ScrapeItem) -> CardSet:async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)return create_set(soup)diff --git a/cyberdrop_dl/crawlers/pornpics.py b/cyberdrop_dl/crawlers/pornpics.pyindex 73dc31873..6e2d527a3 100644--- a/cyberdrop_dl/crawlers/pornpics.py+++ b/cyberdrop_dl/crawlers/pornpics.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -57,7 +58,7 @@ class PornPicsCrawler(Crawler):def update_scrape_item(soup: BeautifulSoup) -> None:selector = "h2" if collection_type == "channel" else "h1"- title = soup.select_one(selector).text.removesuffix(" Nude Pics").removesuffix(" Porn Pics")+ title = css.select_one_get_text(soup, selector).removesuffix(" Nude Pics").removesuffix(" Porn Pics")title = self.create_title(f"{title} [{collection_type}]")scrape_item.setup_as_profile(title)@@ -80,7 +81,7 @@ class PornPicsCrawler(Crawler):soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)scrape_item.url = PRIMARY_URL / "galleries" / gallery_id # canonical URL- title = soup.select_one("h1").text+ title = css.select_one_get_text(soup, "h1")title = self.create_title(title, gallery_id)scrape_item.setup_as_album(title, album_id=gallery_id)@@ -109,7 +110,7 @@ class PornPicsCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, current_page)items = soup.select(IMAGE_SELECTOR)- return soup, tuple(self.parse_url(image.get("href")) for image in items)+ return soup, tuple(self.parse_url(css.get_attr(image, "href")) for image in items)async with self.request_limiter:# The response is JSON but the "content-type" is wrong so we have to request it as textdiff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.pyindex 3b461bce0..808d195ef 100644--- a/cyberdrop_dl/crawlers/porntrex.py+++ b/cyberdrop_dl/crawlers/porntrex.py@@ -10,6 +10,7 @@ from yarl import URLfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:@@ -88,7 +89,7 @@ class PorntrexCrawler(Crawler):if "This album is a private album" in soup.text:raise ScrapeError(401, "Private album")- title = soup.select_one(_SELECTORS.ALBUM_TITLE).text+ title = css.select_one_get_text(soup, _SELECTORS.ALBUM_TITLE)title = self.create_title(title, album_id)scrape_item.setup_as_album(title)@@ -123,13 +124,13 @@ class PorntrexCrawler(Crawler):soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)if "models" in scrape_item.url.parts:- title: str = soup.select_one(_SELECTORS.MODEL_NAME).get_text(strip=True).title()+ title: str = css.select_one_get_text(soup, _SELECTORS.MODEL_NAME).title()elif "members" in scrape_item.url.parts:- title: str = soup.select_one(_SELECTORS.USER_NAME).get_text(strip=True)+ title: str = css.select_one_get_text(soup, _SELECTORS.USER_NAME)elif "latest-updates" in scrape_item.url.parts:title: str = "Latest Updates"else:- title = soup.select_one(_SELECTORS.TITLE).get_text(strip=True)+ title = css.select_one_get_text(soup, _SELECTORS.TITLE)for trash in TITLE_TRASH:title = title.replace(trash, "").strip()@@ -171,6 +172,7 @@ class PorntrexCrawler(Crawler):sort_by: str = scrape_item.url.parts[4] if len(scrape_item.url.parts) > 4 else ""sort_by = sort_by or scrape_item.url.query.get("sort_by") or "post_date"if "search" in scrape_item.url.parts:+ assert len(scrape_item.url.parts) > 3search_query = scrape_item.url.parts[3]block_id = "list_videos_videos"elif "members" in scrape_item.url.parts:diff --git a/cyberdrop_dl/crawlers/postimg.py b/cyberdrop_dl/crawlers/postimg.pyindex b99bb4280..44895c72f 100644--- a/cyberdrop_dl/crawlers/postimg.py+++ b/cyberdrop_dl/crawlers/postimg.py@@ -7,6 +7,7 @@ from yarl import URLfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -64,7 +65,7 @@ class PostImgCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- link_str: str = soup.select_one(DOWNLOAD_BUTTON_SELECTOR).get("href")+ link_str: str = css.select_one_get_attr(soup, DOWNLOAD_BUTTON_SELECTOR, "href")link = self.parse_url(link_str).with_query(None)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.pyindex 549744df3..e917d07ba 100644--- a/cyberdrop_dl/crawlers/redgifs.py+++ b/cyberdrop_dl/crawlers/redgifs.py@@ -78,7 +78,7 @@ class RedGifsCrawler(Crawler):scrape_item.possible_datetime = json_resp["gif"]["createDate"]links: dict[str, str] = json_resp["gif"]["urls"]- link_str: str = links.get("hd") or links.get("sd")+ link_str: str = links.get("hd") or links["sd"]link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/rule34vault.py b/cyberdrop_dl/crawlers/rule34vault.pyindex 8cbbb45f2..2ced895d2 100644--- a/cyberdrop_dl/crawlers/rule34vault.py+++ b/cyberdrop_dl/crawlers/rule34vault.py@@ -86,7 +86,8 @@ class Rule34VaultCrawler(Crawler):scrape_item.url = canonical_urlmedia_tag = soup.select_one(_SELECTORS.VIDEO) or soup.select_one(_SELECTORS.IMAGE)- link_str: str = media_tag["src"]+ assert media_tag+ link_str: str = css.get_attr(media_tag, "src")for trash in (".small", ".thumbnail", ".picsmall", ".720", ".hevc"):link_str = link_str.replace(trash, "")diff --git a/cyberdrop_dl/crawlers/rule34video.py b/cyberdrop_dl/crawlers/rule34video.pyindex ac43e206f..1761bd55a 100644--- a/cyberdrop_dl/crawlers/rule34video.py+++ b/cyberdrop_dl/crawlers/rule34video.py@@ -115,7 +115,7 @@ class Rule34VideoCrawler(Crawler):def get_video_info(soup: BeautifulSoup) -> VideoInfo:- title = soup.select_one(VIDEO_TITLE_SELECTOR).text.strip()+ title = css.select_one_get_text(soup, VIDEO_TITLE_SELECTOR)info_js_script = soup.select_one(JS_SELECTOR)info: dict[str, str | dict] = javascript.parse_json_to_dict(info_js_script.text)info["title"] = title@@ -126,7 +126,7 @@ def get_video_info(soup: BeautifulSoup) -> VideoInfo:def get_available_formats(soup: BeautifulSoup) -> Generator[Format]:for download in soup.select(DOWNLOADS_SELECTOR):- link_str: str = download.get("href")+ link_str: str = css.get_attr(download, "href")if "/tags/" in link_str or not all(p in link_str for p in REQUIRED_FORMAT_STRINGS):continueext, res = download.text.rsplit(" ", 1)@@ -150,10 +150,10 @@ def get_playlist_title(soup: BeautifulSoup, playlist_type: str) -> str:selector = PLAYLIST_TITLE_SELECTORS[playlist_type]title_tag = css.select_one(soup, selector)if playlist_type in ("tags", "search"):- for span in title_tag.find_all("span"):+ for span in title_tag.select("span"):span.decompose()- title = css.get_text(title_tag.text)+ title = css.get_text(title_tag)for trash in TITLE_TRASH:title = title.replace(trash, "").strip()diff --git a/cyberdrop_dl/crawlers/rule34xxx.py b/cyberdrop_dl/crawlers/rule34xxx.pyindex b643f91c4..970edfb73 100644--- a/cyberdrop_dl/crawlers/rule34xxx.py+++ b/cyberdrop_dl/crawlers/rule34xxx.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -66,7 +67,7 @@ class Rule34XXXCrawler(Crawler):if date_tag := soup.select_one(_SELECTOR.DATE):date_str = date_tag.get_text(strip=True).removeprefix("Posted: ")scrape_item.possible_datetime = self.parse_date(date_str)- link_str: str = media_tag["src"]+ link_str: str = css.get_attr(media_tag, "src")link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)diff --git a/cyberdrop_dl/crawlers/thisvid.py b/cyberdrop_dl/crawlers/thisvid.pyindex ee3613106..e4da4230b 100644--- a/cyberdrop_dl/crawlers/thisvid.py+++ b/cyberdrop_dl/crawlers/thisvid.py@@ -158,7 +158,7 @@ class ThisVidCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- js_text: str = soup.select_one(_SELECTORS.ALBUM_ID).text+ js_text: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_ID)album_id: str = get_text_between(js_text, "params['album_id'] =", ";").strip()results = await self.get_album_results(album_id)title: str = css.select_one_get_text(soup, _SELECTORS.ALBUM_NAME)diff --git a/cyberdrop_dl/crawlers/tokyomotion.py b/cyberdrop_dl/crawlers/tokyomotion.pyindex 8d75e4303..398b2f78e 100644--- a/cyberdrop_dl/crawlers/tokyomotion.py+++ b/cyberdrop_dl/crawlers/tokyomotion.py@@ -1,12 +1,12 @@from __future__ import annotations-import contextlibimport refrom typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths+from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_partsif TYPE_CHECKING:@@ -76,22 +76,17 @@ class TokioMotionCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- with contextlib.suppress(AttributeError):- relative_date_str = soup.select_one(self.video_date_selector).text.strip()- scrape_item.possible_datetime = self.parse_date(relative_date_str)+ scrape_item.possible_datetime = self.parse_date(css.select_one_get_text(soup, self.video_date_selector))- try:- srcSD = soup.select_one('source[title="SD"]')- srcHD = soup.select_one('source[title="HD"]')- src = srcHD or srcSD- link_str: str = src.get("src")- link = self.parse_url(link_str)- except AttributeError:+ src = soup.select_one('source[title="SD"]') or soup.select_one('source[title="HD"]')+ if not src:if "This is a private" in soup.text:- raise ScrapeError(401, "Private video") from None- raise ScrapeError(422, "Couldn't find video source") from None+ raise ScrapeError(401, "Private video")+ raise ScrapeError(422, "Couldn't find video source")- title = soup.select_one("title").text.rsplit(" - TOKYO Motion")[0].strip()+ link_str: str = css.get_attr(src, "src")+ link = self.parse_url(link_str)+ title = css.select_one_get_text(soup, "title").rsplit(" - TOKYO Motion")[0].strip()filename, ext = f"{video_id}.mp4", ".mp4"custom_filename, _ = self.get_filename_and_ext(f"{title} [{video_id}]{ext}")await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)@@ -103,15 +98,15 @@ class TokioMotionCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- try:- img = soup.select_one(self.image_selector)- link_str: str = img.get("src")- link = self.parse_url(link_str)- except AttributeError:++ img = soup.select_one(self.image_selector)+ if not img:if "This is a private" in soup.text:- raise ScrapeError(401, "Private Photo") from None- raise ScrapeError(422, "Couldn't find image source") from None+ raise ScrapeError(401, "Private Photo")+ raise ScrapeError(422, "Couldn't find image source")+ link_str: str = css.get_attr(img, "src")+ link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)@@ -206,7 +201,7 @@ class TokioMotionCrawler(Crawler):return scrape_item.url.parts[3]async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- return soup.select_one(self.album_title_selector).get_text()+ return css.select_one_get_text(soup, self.album_title_selector)def add_user_title(self, scrape_item: ScrapeItem) -> None:try:diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex 26488e395..71422830a 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -79,7 +79,7 @@ from cyberdrop_dl.types import AnyDict, U32Int, U32IntArray, U32IntSequencefrom cyberdrop_dl.utils.logger import logif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator+ from collections.abc import AsyncGenerator, Generator, Mappingfrom yarl import URL@@ -742,7 +742,7 @@ class MegaApi:nodes = {node["h"]: node async for node in prepare_nodes()}return nodes- async def _build_file_system(self, nodes_map: dict[str, Node], root_ids: list[str]) -> dict[Path, Node]:+ async def _build_file_system(self, nodes_map: Mapping[str, Node], root_ids: list[str]) -> dict[Path, Node]:"""Builds a flattened dictionary representing a file system from a list of items.Returns:diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 845f80a86..a9d4f4017 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -28,7 +28,7 @@ def select_one(tag: Tag, selector: str) -> Tag:def select_one_get_text(tag: Tag, selector: str, strip: bool = True) -> str:- """Same as `tag.select_one` but asserts the result is not `None`"""+ """Same as `tag.select_one.get_text(strip=strip)` but asserts the result is not `None`"""return get_text(select_one(tag, selector), strip)