Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add API for single tag text extraction
NTFSvolumefb44c0d962be
diff --git a/cyberdrop_dl/crawlers/turbovid.py b/cyberdrop_dl/crawlers/turbovid.pyindex dadb602d4..a8f9774dc 100644--- a/cyberdrop_dl/crawlers/turbovid.py+++ b/cyberdrop_dl/crawlers/turbovid.py@@ -4,9 +4,6 @@ import asyncioimport dataclassesfrom typing import TYPE_CHECKING, Any, ClassVar, final, override-from bs4 import BeautifulSoup-from bs4.filter import SoupStrainer-from cyberdrop_dl.crawlers import Registryfrom cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -137,6 +134,6 @@ class Album:def _extract_album(html: str) -> Album:return Album(id=extr_text(html, 'const ALBUM_ENC_ID = "', '"'),- name=css.select_text(BeautifulSoup(html, "html.parser", parse_only=SoupStrainer("h1")), "h1"),+ name=css.select_tag_text(html, "h1"),files=tuple(TextExtractor(extr_text(html, "const FILES = [", "],")).repeat('{ id:"', '"')),)diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 4369342d7..8f6cf4e41 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -6,6 +6,7 @@ import htmlfrom typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast, overloadfrom bs4 import BeautifulSoup+from bs4.filter import SoupStrainerfrom cyberdrop_dl.exceptions import ScrapeError@@ -68,6 +69,10 @@ def select_text(tag: Tag, selector: str, *, strip: bool = True, decompose: str |return text(inner_tag, strip=strip)+def select_tag_text(html: str, tag_name: str) -> str:+ return select_text(soup(html, parse_only=tag_name), tag_name)++def attr_or_none(tag: Tag, attribute: str) -> str | None:"""Same as `tag.get(attribute)` but asserts the result is a single str"""attribute_ = attribute@@ -199,12 +204,14 @@ def parse_form(form: Tag, /) -> HTMLForm:return HTMLForm(cast("HttpMethod", method), action, inputs)-def soup(content: str) -> BeautifulSoup:- return BeautifulSoup(content, "html.parser")+def soup(content: str, parse_only: tuple[str, ...] | str | None = None) -> BeautifulSoup:+ return BeautifulSoup(+ content, "html.parser", parse_only=SoupStrainer(parse_only) if parse_only is not None else None+ )-async def asoup(content: str) -> BeautifulSoup:- return await asyncio.to_thread(soup, content)+async def asoup(content: str, parse_only: tuple[str, ...] | str | None = None) -> BeautifulSoup:+ return await asyncio.to_thread(soup, content, parse_only)unescape = html.unescape