Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add API for single tag text extraction

NTFSvolumefb44c0d962be
diff --git a/cyberdrop_dl/crawlers/turbovid.py b/cyberdrop_dl/crawlers/turbovid.py
index dadb602d4..a8f9774dc 100644
--- a/cyberdrop_dl/crawlers/turbovid.py
+++ b/cyberdrop_dl/crawlers/turbovid.py
@@ -4,9 +4,6 @@ import asyncio
import dataclasses
from typing import TYPE_CHECKING, Any, ClassVar, final, override
-from bs4 import BeautifulSoup
-from bs4.filter import SoupStrainer
-
from cyberdrop_dl.crawlers import Registry
from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -137,6 +134,6 @@ class Album:
def _extract_album(html: str) -> Album:
return Album(
id=extr_text(html, 'const ALBUM_ENC_ID = "', '"'),
- name=css.select_text(BeautifulSoup(html, "html.parser", parse_only=SoupStrainer("h1")), "h1"),
+ name=css.select_tag_text(html, "h1"),
files=tuple(TextExtractor(extr_text(html, "const FILES = [", "],")).repeat('{ id:"', '"')),
)
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 4369342d7..8f6cf4e41 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -6,6 +6,7 @@ import html
from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast, overload
from bs4 import BeautifulSoup
+from bs4.filter import SoupStrainer
from cyberdrop_dl.exceptions import ScrapeError
@@ -68,6 +69,10 @@ def select_text(tag: Tag, selector: str, *, strip: bool = True, decompose: str |
return text(inner_tag, strip=strip)
+def select_tag_text(html: str, tag_name: str) -> str:
+ return select_text(soup(html, parse_only=tag_name), tag_name)
+
+
def attr_or_none(tag: Tag, attribute: str) -> str | None:
"""Same as `tag.get(attribute)` but asserts the result is a single str"""
attribute_ = attribute
@@ -199,12 +204,14 @@ def parse_form(form: Tag, /) -> HTMLForm:
return HTMLForm(cast("HttpMethod", method), action, inputs)
-def soup(content: str) -> BeautifulSoup:
- return BeautifulSoup(content, "html.parser")
+def soup(content: str, parse_only: tuple[str, ...] | str | None = None) -> BeautifulSoup:
+ return BeautifulSoup(
+ content, "html.parser", parse_only=SoupStrainer(parse_only) if parse_only is not None else None
+ )
-async def asoup(content: str) -> BeautifulSoup:
- return await asyncio.to_thread(soup, content)
+async def asoup(content: str, parse_only: tuple[str, ...] | str | None = None) -> BeautifulSoup:
+ return await asyncio.to_thread(soup, content, parse_only)
unescape = html.unescape