Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove redundant utils

NTFSvolume918f41840c4c
diff --git a/cyberdrop_dl/aio.py b/cyberdrop_dl/aio.py
index ab15b729e..7b8f1ffc9 100644
--- a/cyberdrop_dl/aio.py
+++ b/cyberdrop_dl/aio.py
@@ -8,30 +8,15 @@ import dataclasses
import shutil
import sys
import tempfile
-from abc import ABC, abstractmethod
from pathlib import Path
from stat import S_ISREG
-from typing import (
- IO,
- TYPE_CHECKING,
- Any,
- AnyStr,
- Generic,
- ParamSpec,
- Self,
- TypeVar,
- TypeVarTuple,
- cast,
- final,
- overload,
-)
+from typing import IO, TYPE_CHECKING, Any, AnyStr, Generic, ParamSpec, Self, TypeVar, TypeVarTuple, cast, overload
from weakref import WeakValueDictionary
from typing_extensions import Sentinel
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterable, Iterator
- from types import TracebackType
from _typeshed import OpenBinaryMode, OpenTextMode
@@ -116,38 +101,6 @@ class AsyncIteratorWrapper(Generic[_T]):
return cast("_T", value)
-class AsyncContextManagerMixin(ABC):
- __stack: contextlib.AsyncExitStack | None = None
-
- @final
- @property
- def _stack(self) -> contextlib.AsyncExitStack[bool | None]:
- if self.__stack is None:
- raise RuntimeError(f"{type(self).__name__} can only be used as a context manager")
- return self.__stack
-
- @final
- async def __aenter__(self) -> Self:
- if self.__stack is not None:
- raise RuntimeError(f"{type(self).__name__} does not allow reentrance")
- self.__stack = contextlib.AsyncExitStack()
- _ = await self.__stack.__aenter__()
- await self._async_ctx_()
- return self
-
- @final
- async def __aexit__(
- self,
- exc_type: type[BaseException] | None,
- exc_val: BaseException | None,
- exc_tb: TracebackType | None,
- ) -> bool | None:
- return await self._stack.__aexit__(exc_type, exc_val, exc_tb)
-
- @abstractmethod
- async def _async_ctx_(self) -> None: ...
-
-
async def gather(*coros: Awaitable[_T]) -> list[_T]:
"""Like asyncio.gather but an exception on any coro cancels all pending coros
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index c109b1e2e..3b7048bcd 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -22,7 +22,7 @@ from bs4 import BeautifulSoup, Tag
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import LoginError, MaxChildrenError, ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, is_blob_or_svg
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, is_blob_or_svg
from cyberdrop_dl.utils.dates import TimeStamp, to_timestamp
if TYPE_CHECKING:
@@ -535,7 +535,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
def _lazy_load_embeds(self, post: ForumPostProtocol) -> Iterable[str]:
selector = self.SELECTORS.posts.lazy_load_embeds
for lazy_media in css.iselect(post.content, selector.element):
- yield get_text_between(css.attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")
+ yield extr_text(css.attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")
async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:
async for soup in self.web_pager(scrape_item.url, self.get_next_page):
diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.py
index 5b7538e54..016277670 100644
--- a/cyberdrop_dl/crawlers/_kvs.py
+++ b/cyberdrop_dl/crawlers/_kvs.py
@@ -11,7 +11,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPaths
from cyberdrop_dl.exceptions import DownloadError, ScrapeError
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, open_graph, parse_url
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, open_graph, parse_url
if TYPE_CHECKING:
from collections.abc import Sequence
@@ -190,7 +190,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):
soup = await self.request_soup(scrape_item.url)
if not album_id:
js_text = css.select_text(soup, Selector.ALBUM_ID)
- album_id = get_text_between(js_text, "params['album_id'] =", ";")
+ album_id = extr_text(js_text, "params['album_id'] =", ";")
results = await self.get_album_results(album_id)
title = css.select_text(soup, Selector.ALBUM_NAME)
diff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.py
index e99feae70..9a102661d 100644
--- a/cyberdrop_dl/crawlers/_yetishare.py
+++ b/cyberdrop_dl/crawlers/_yetishare.py
@@ -7,7 +7,7 @@ from bs4 import BeautifulSoup
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id
from cyberdrop_dl.exceptions import DDOSGuardError, PasswordProtectedError, ScrapeError
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text
if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
@@ -80,7 +80,7 @@ class YetiShareCrawler(Crawler, is_abc=True):
else:
# ex: loadImages('folder', '12345', 1, 0, '', {'searchTerm': "", 'filterUploadedDateRange': ""});
page_type = "folder"
- load_images = get_text_between(soup.select(Selector.LOAD_IMAGES)[-1].text, "loadImages(", ");")
+ load_images = extr_text(soup.select(Selector.LOAD_IMAGES)[-1].text, "loadImages(", ");")
node_id = load_images.replace("'", "").split(",")[1].strip()
page = 1
@@ -135,7 +135,7 @@ class YetiShareCrawler(Crawler, is_abc=True):
_check_is_available(soup)
content_id = int(
- get_text_between(
+ extr_text(
css.select_text(soup, Selector.FILE_INFO),
"showFileInformation(",
");",
@@ -156,7 +156,7 @@ class YetiShareCrawler(Crawler, is_abc=True):
# Manually parse link. Some URLs are invalid. ex: https://cyberfile.me/7cfu
# For the download URL, the slug does not actually matter. It can be anything
- raw_link = get_text_between(css.attr(download_tag, "onclick"), "('", "');")
+ raw_link = extr_text(css.attr(download_tag, "onclick"), "('", "');")
token = raw_link.rpartition("?download_token=")[-1]
link = self.parse_url(raw_link).with_query(download_token=token)
diff --git a/cyberdrop_dl/crawlers/archivebate.py b/cyberdrop_dl/crawlers/archivebate.py
index b3e6ff8f9..52e074029 100644
--- a/cyberdrop_dl/crawlers/archivebate.py
+++ b/cyberdrop_dl/crawlers/archivebate.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, open_graph
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, open_graph
if TYPE_CHECKING:
from cyberdrop_dl.crawlers.crawler import SupportedPaths
@@ -44,7 +44,7 @@ class ArchiveBateCrawler(Crawler):
if "This video has been deleted" in soup.get_text():
raise ScrapeError(410)
- upload_date = get_text_between(open_graph.description(soup), "show on", " - ").strip()
+ upload_date = extr_text(open_graph.description(soup), "show on", " - ").strip()
user_name = css.select_text(soup, Selector.USER_NAME)
site_name = css.select_text(soup, Selector.SITE_NAME)
scrape_item.setup_as_profile(self.create_title(f"{user_name} [{site_name}]"))
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.py
index 2cbe7f426..bf11c939d 100644
--- a/cyberdrop_dl/crawlers/ashemaletube.py
+++ b/cyberdrop_dl/crawlers/ashemaletube.py
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, m3u8
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, m3u8
if TYPE_CHECKING:
from bs4 import BeautifulSoup, Tag
@@ -151,7 +151,7 @@ class AShemaleTubeCrawler(Crawler):
link_str: str = css.attr(image, "src")
else:
style: str = css.select(img_tag, "a", "style")
- link_str = get_text_between(style, "url('", "');")
+ link_str = extr_text(style, "url('", "');")
url = self.parse_url(link_str).with_query(None)
filename, ext = self.get_filename_and_ext(url.name)
custom_filename = self.create_custom_filename(filename, ext, file_id=css.attr(img_tag, "data-image-id"))
@@ -190,7 +190,7 @@ class AShemaleTubeCrawler(Crawler):
)
async def parse_player_info(self, script_text: str) -> tuple[Resolution, AbsoluteHttpURL, m3u8.Rendition]:
- sources = get_text_between(script_text, "sources: ", "aspectRatio").strip().strip(",")
+ sources = extr_text(script_text, "sources: ", "aspectRatio").strip().strip(",")
sources_data = json.loads(sources)
url = self.parse_url(sources_data["hlsAuto"])
diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.py
index 4d14e3f80..e9702006a 100644
--- a/cyberdrop_dl/crawlers/doodstream.py
+++ b/cyberdrop_dl/crawlers/doodstream.py
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text
if TYPE_CHECKING:
from bs4 import BeautifulSoup
@@ -96,7 +96,7 @@ class DoodStreamCrawler(Crawler):
def _md5_pass(soup: BeautifulSoup) -> str:
js_text = css.select_text(soup, Selector.MD5_JS)
- return get_text_between(js_text, "/pass_md5/", "'")
+ return extr_text(js_text, "/pass_md5/", "'")
def _file_id(soup: BeautifulSoup) -> str:
diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.py
index fb0f49961..5102b98e0 100644
--- a/cyberdrop_dl/crawlers/eporner.py
+++ b/cyberdrop_dl/crawlers/eporner.py
@@ -9,7 +9,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text
if TYPE_CHECKING:
from bs4 import BeautifulSoup, Tag
@@ -218,7 +218,7 @@ def _parse_video(soup: BeautifulSoup) -> Video:
formats = [VideoSource.parse(tag) for tag in soup.select(Selector.FORMATS)]
return Video(
- title=get_text_between(ld_json, 'name": "', '",'),
- date=get_text_between(ld_json, 'uploadDate": "', '"'),
+ title=extr_text(ld_json, 'name": "', '",'),
+ date=extr_text(ld_json, 'uploadDate": "', '"'),
best_src=max(formats),
)
diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.py
index 0b97e5ad4..62afb9225 100644
--- a/cyberdrop_dl/crawlers/filester.py
+++ b/cyberdrop_dl/crawlers/filester.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, open_graph
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, open_graph
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
@@ -54,7 +54,7 @@ class FilesterCrawler(Crawler):
scrape_item.setup_as_album(title, album_id=album_id)
for on_click in css.iselect(soup, Selector.FILES, "onclick"):
- web_url = self.parse_url(get_text_between(on_click, "'", "'"))
+ web_url = self.parse_url(extr_text(on_click, "'", "'"))
new_scrape_item = scrape_item.create_child(web_url)
self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
diff --git a/cyberdrop_dl/crawlers/flickr.py b/cyberdrop_dl/crawlers/flickr.py
index 28462ddb4..a96a36443 100644
--- a/cyberdrop_dl/crawlers/flickr.py
+++ b/cyberdrop_dl/crawlers/flickr.py
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import error_handling_context, error_handling_wrapper, get_text_between
+from cyberdrop_dl.utils import error_handling_context, error_handling_wrapper, extr_text
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
@@ -106,7 +106,7 @@ class FlickrAPI:
self._crawler.disable_on_error("Unable to get public API key"),
):
text = await self._crawler.request_text(prints)
- self.api_key = get_text_between(text, 'flickr.api.site_key = "', '"')
+ self.api_key = extr_text(text, 'flickr.api.site_key = "', '"')
async def _request(self, method: str, **params: Any) -> dict[str, Any]:
api_url = self.API_ENDPOINT.with_query(
diff --git a/cyberdrop_dl/crawlers/fucking_fast.py b/cyberdrop_dl/crawlers/fucking_fast.py
index 9d873025f..91ba83a8d 100644
--- a/cyberdrop_dl/crawlers/fucking_fast.py
+++ b/cyberdrop_dl/crawlers/fucking_fast.py
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text
if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
@@ -34,7 +34,7 @@ class FuckingFastCrawler(Crawler):
soup = await self.request_soup(scrape_item.url)
name = css.page_title(soup)
- dl_link = get_text_between(str(soup), 'window.open("https:', ")").strip('"')
+ dl_link = extr_text(str(soup), 'window.open("https:', ")").strip('"')
link = self.parse_url(f"https:{dl_link}")
filename, ext = self.get_filename_and_ext(name)
await self.handle_file(scrape_item.url, scrape_item, name, ext, debrid_link=link, custom_filename=filename)
diff --git a/cyberdrop_dl/crawlers/gupload.py b/cyberdrop_dl/crawlers/gupload.py
index c117b7add..e74b07e29 100644
--- a/cyberdrop_dl/crawlers/gupload.py
+++ b/cyberdrop_dl/crawlers/gupload.py
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import error_handling_wrapper, get_text_between, xor_decrypt
+from cyberdrop_dl.utils import error_handling_wrapper, extr_text, xor_decrypt
if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
@@ -61,5 +61,5 @@ class GUploadCrawler(Crawler):
async def _request_video_config(self, url: AbsoluteHttpURL) -> dict[str, Any]:
html = await self.request_text(url, impersonate=True)
- config_text = get_text_between(html, "var _cfg = ", "');").partition("('")[-1]
+ config_text = extr_text(html, "var _cfg = ", "');").partition("('")[-1]
return _decode_config(config_text)
diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.py
index 0e8aec83e..1071bb350 100644
--- a/cyberdrop_dl/crawlers/imgur.py
+++ b/cyberdrop_dl/crawlers/imgur.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text
if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
@@ -41,7 +41,7 @@ class ImgurCrawler(Crawler):
soup = await self.request_soup(self.PRIMARY_URL)
js_src = css.select(soup, "script[src*='/desktop-assets/js/main']", "src")
js_text = await self.request_text(self.parse_url(js_src))
- self.client_id = get_text_between(js_text, 'apiClientId:"', '"')
+ self.client_id = extr_text(js_text, 'apiClientId:"', '"')
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.host == _IMAGE_CDN.host:
diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.py
index 14d883553..b81af0e8e 100644
--- a/cyberdrop_dl/crawlers/kemono.py
+++ b/cyberdrop_dl/crawlers/kemono.py
@@ -17,7 +17,7 @@ from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError
from cyberdrop_dl.models import AliasModel
from cyberdrop_dl.models.validators import falsy_as, falsy_as_none
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, remove_parts
+from cyberdrop_dl.utils import css, error_handling_wrapper
from cyberdrop_dl.utils.dates import to_timestamp
if TYPE_CHECKING:
@@ -622,7 +622,7 @@ class KemonoCrawler(KemonoBaseCrawler):
def _thumbnail_to_src(og_url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- url = remove_parts(og_url, "thumbnails", "thumbnail").with_query(None)
+ url = og_url.with_path(og_url.path.replace("/thumbnails/", "/").replace("/thumbnail/", "/"))
if name := og_url.query.get("f"):
return url.with_query(f=name)
return url
diff --git a/cyberdrop_dl/crawlers/leakedzone.py b/cyberdrop_dl/crawlers/leakedzone.py
index db966b49f..83b5912b3 100644
--- a/cyberdrop_dl/crawlers/leakedzone.py
+++ b/cyberdrop_dl/crawlers/leakedzone.py
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.compat import IntEnum
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text
if TYPE_CHECKING:
from bs4 import BeautifulSoup
@@ -75,7 +75,7 @@ class LeakedZoneCrawler(Crawler):
@classmethod
def get_encoded_video_url(cls, soup: BeautifulSoup) -> str:
js_text = css.select_text(soup, _SELECTORS.JW_PLAYER)
- return get_text_between(js_text, 'file: f("', '"),')
+ return extr_text(js_text, 'file: f("', '"),')
@error_handling_wrapper
async def model(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/crawlers/noodle_magazine.py b/cyberdrop_dl/crawlers/noodle_magazine.py
index e20cfe79e..27bfe7adf 100644
--- a/cyberdrop_dl/crawlers/noodle_magazine.py
+++ b/cyberdrop_dl/crawlers/noodle_magazine.py
@@ -9,7 +9,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, parse_url
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, parse_url
if TYPE_CHECKING:
from collections.abc import Generator
@@ -109,7 +109,7 @@ def _parse_video(soup: BeautifulSoup) -> Video:
except css.SelectorError:
raise ScrapeError(404) from None
- playlist = json.loads(get_text_between(playlist_js, "window.playlist = ", ";\nwindow.ads"))
+ playlist = json.loads(extr_text(playlist_js, "window.playlist = ", ";\nwindow.ads"))
resolution, src = max(_parse_sources(playlist["sources"]))
content_url = parse_url(json_ld["contentUrl"])
diff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.py
index e525ed9de..f3c114a7d 100644
--- a/cyberdrop_dl/crawlers/odnoklassniki.py
+++ b/cyberdrop_dl/crawlers/odnoklassniki.py
@@ -7,7 +7,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, json
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, json
if TYPE_CHECKING:
from bs4 import BeautifulSoup
@@ -80,7 +80,7 @@ class OdnoklassnikiCrawler(Crawler):
soup = await self.request_soup(scrape_item.url, headers=_HEADERS)
channel_id = channel_str.removeprefix("c")
- gwt_hash = get_text_between(css.select_text(soup, Selector.CHANNEL_HASH), 'gwtHash:"', '",')
+ gwt_hash = extr_text(css.select_text(soup, Selector.CHANNEL_HASH), 'gwtHash:"', '",')
try:
last_element_id = css.select(soup, *Selector.CHANNEL_LAST_ELEMENT)
diff --git a/cyberdrop_dl/crawlers/patreon.py b/cyberdrop_dl/crawlers/patreon.py
index de1e68e98..ed937a02b 100644
--- a/cyberdrop_dl/crawlers/patreon.py
+++ b/cyberdrop_dl/crawlers/patreon.py
@@ -10,7 +10,7 @@ from typing_extensions import ReadOnly
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, next_js
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, next_js
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Generator
@@ -214,7 +214,7 @@ class PatreonCrawler(Crawler):
def _extract_campaign_id(soup: BeautifulSoup):
- return get_text_between(str(soup), r"{\"value\":{\"campaign\":{\"data\":{\"id\":\"", r"\"")
+ return extr_text(str(soup), r"{\"value\":{\"campaign\":{\"data\":{\"id\":\"", r"\"")
def _extract_campaign_id_next_js(soup: BeautifulSoup) -> str:
diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.py
index 7d3a937dd..f5029008f 100644
--- a/cyberdrop_dl/crawlers/pornhub.py
+++ b/cyberdrop_dl/crawlers/pornhub.py
@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal, NamedTuple, TypedDict
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text
if TYPE_CHECKING:
from bs4 import BeautifulSoup
@@ -301,12 +301,12 @@ class PornHubCrawler(Crawler):
def get_upload_date_str(soup: BeautifulSoup) -> str:
date_text = css.select_text(soup, _SELECTORS.DATE)
- return get_text_between(date_text, 'uploadDate": "', '",')
+ return extr_text(date_text, 'uploadDate": "', '",')
def get_media_list(soup: BeautifulSoup) -> list[Media]:
flashvars: str = css.select(soup, _SELECTORS.JS_VIDEO_INFO).text
- media_text = get_text_between(flashvars, '"mediaDefinitions":', '"isVertical"').strip().removesuffix(",")
+ media_text = extr_text(flashvars, '"mediaDefinitions":', '"isVertical"').strip().removesuffix(",")
return json.loads(media_text)
diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.py
index 5a3066c09..e3ca5e32f 100644
--- a/cyberdrop_dl/crawlers/sendvid.py
+++ b/cyberdrop_dl/crawlers/sendvid.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, remove_parts
+from cyberdrop_dl.utils import css, error_handling_wrapper
if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
@@ -67,7 +67,7 @@ class SendVidCrawler(Crawler):
return PRIMARY_URL.with_path(video_id)
if "embed" in url.parts:
- return remove_parts(url, "embed")
+ return url.with_path(url.path.replace("/embed/", "/"), keep_query=True, keep_fragment=True)
return url
diff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.py
index 8d1058b7e..50a717a49 100644
--- a/cyberdrop_dl/crawlers/spankbang.py
+++ b/cyberdrop_dl/crawlers/spankbang.py
@@ -7,7 +7,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, json
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, json
if TYPE_CHECKING:
from collections.abc import Generator
@@ -158,13 +158,13 @@ def _parse_video(soup: BeautifulSoup, display_id: str) -> Video:
title_tag = css.select(soup, "div#video h1")
stream_js_text = css.select_text(soup, Selector.STREAM_DATA)
- stream_data = get_text_between(stream_js_text, "stream_data = ", ";")
+ stream_data = extr_text(stream_js_text, "stream_data = ", ";")
res, url = max(_parse_formats(stream_data))
return Video(
id=display_id,
resolution=res,
url=url,
- stream_id=get_text_between(stream_js_text, "ana_video_id = ", ";").strip("'"),
+ stream_id=extr_text(stream_js_text, "ana_video_id = ", ";").strip("'"),
stream_key=css.select(soup, "[data-streamkey]", "data-streamkey"),
title=css.attr_or_none(title_tag, "title") or css.text(title_tag),
)
diff --git a/cyberdrop_dl/crawlers/streamtape.py b/cyberdrop_dl/crawlers/streamtape.py
index 9ee95d119..f3a9e048a 100644
--- a/cyberdrop_dl/crawlers/streamtape.py
+++ b/cyberdrop_dl/crawlers/streamtape.py
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, open_graph, parse_url
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, open_graph, parse_url
if TYPE_CHECKING:
from bs4 import BeautifulSoup
@@ -50,6 +50,6 @@ class StreamtapeCrawler(Crawler):
def _extract_download_link(soup: BeautifulSoup) -> AbsoluteHttpURL:
script = css.select_text(soup, Selector.JS_TOKEN)
- token = get_text_between(script, "&token=", "'")
+ token = extr_text(script, "&token=", "'")
bait_url = css.select_text(soup, Selector.BAIT_LINK)
return parse_url(f"https:/{bait_url}").update_query(token=token)
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.py
index fc44e64b6..bb0c0130a 100644
--- a/cyberdrop_dl/crawlers/xhamster.py
+++ b/cyberdrop_dl/crawlers/xhamster.py
@@ -12,7 +12,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.mediaprops import Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import error_handling_wrapper, get_text_between, parse_url, xor_decrypt
+from cyberdrop_dl.utils import error_handling_wrapper, extr_text, parse_url, xor_decrypt
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
@@ -238,7 +238,7 @@ class XhamsterCrawler(Crawler):
async def _get_window_initials(self, url: AbsoluteHttpURL) -> dict[str, Any]:
self._disable_ai_title_translations(url)
content = await self.request_text(url)
- initials = get_text_between(content, "window.initials=", ";</script>")
+ initials = extr_text(content, "window.initials=", ";</script>")
return json.loads(initials)
diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.py
index 6801f1dc7..5830c53a9 100644
--- a/cyberdrop_dl/crawlers/xvideos.py
+++ b/cyberdrop_dl/crawlers/xvideos.py
@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text
if TYPE_CHECKING:
from bs4 import BeautifulSoup
@@ -120,7 +120,7 @@ class XVideosCrawler(Crawler):
title = css.page_title(soup, self.DOMAIN)
scrape_item.uploaded_at = self.parse_iso_date(css.json_ld(soup)["uploadDate"])
script = css.select_text(soup, Selectors.HLS_VIDEO_JS)
- m3u8_url = self.parse_url(get_text_between(script, "setVideoHLS('", "')"))
+ m3u8_url = self.parse_url(extr_text(script, "setVideoHLS('", "')"))
m3u8, info = await self.request_m3u8_playlist(m3u8_url)
custom_filename = self.create_custom_filename(title, ".mp4", file_id=encoded_id, resolution=info.resolution)
# Remove url slug to prevent duplicates in database. It's language specific and not required.
@@ -135,14 +135,14 @@ class XVideosCrawler(Crawler):
soup = await self._get_soup(scrape_item.url)
script = css.select_text(soup, Selectors.ACCOUNT_INFO_JS)
- display_name = get_text_between(script, '"display":"', '",')
+ display_name = extr_text(script, '"display":"', '",')
scrape_item.setup_as_profile(self.create_title(f"{display_name} [{name}]"))
frag = scrape_item.url.fragment
part = "photos" if "profiles" in scrape_item.url.parts else "post"
if not frag or "_tabPhotos" in frag:
galleries: dict[str, Any] | list[str] = json.loads(
- get_text_between(script, '"galleries":', '"visitor":').removesuffix(",")
+ extr_text(script, '"galleries":', '"visitor":').removesuffix(",")
)
if isinstance(galleries, dict):
@@ -158,11 +158,11 @@ class XVideosCrawler(Crawler):
await self._iter_api_pages(scrape_item, scrape_item.url / "videos/new", "videos")
if not frag or "quickies" in frag:
- has_quickies = get_text_between(script, '"has_quickies":', ",").strip()
+ has_quickies = extr_text(script, '"has_quickies":', ",").strip()
if has_quickies == "false":
return
- user_id = get_text_between(script, '"id_user":', ",").strip()
+ user_id = extr_text(script, '"id_user":', ",").strip()
quickies_api = scrape_item.url.origin() / "quickies-api/profilevideos/all/none/N" / user_id
await self._iter_api_pages(scrape_item, quickies_api, "quickies")
diff --git a/cyberdrop_dl/crawlers/youjizz.py b/cyberdrop_dl/crawlers/youjizz.py
index 8ccee28ab..5964786a2 100644
--- a/cyberdrop_dl/crawlers/youjizz.py
+++ b/cyberdrop_dl/crawlers/youjizz.py
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, open_graph
+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, open_graph
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -76,7 +76,7 @@ class YouJizzCrawler(Crawler):
def _parse_video(soup: BeautifulSoup) -> Video:
js_text = css.select_text(soup, JS_SELECTOR)
- encodings_text = get_text_between(js_text, "var dataEncodings =", "var encodings").strip().removesuffix(";")
+ encodings_text = extr_text(js_text, "var dataEncodings =", "var encodings").strip().removesuffix(";")
data_encodings = json.loads(encodings_text)
return Video(
title=open_graph.title(soup),
diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.py
index 3d3f8df2b..05164a7ef 100644
--- a/cyberdrop_dl/utils/__init__.py
+++ b/cyberdrop_dl/utils/__init__.py
@@ -279,11 +279,11 @@ def _check_for_partial_files(path: Path) -> None:
logger.warning("There are partial downloads in the downloads folder")
-def get_text_between(original_text: str, start: str, end: str) -> str:
+def extr_text(text: str, /, start: str, end: str) -> str:
"""Extracts the text between two strings in a larger text. Result will be stripped"""
- start_index = original_text.index(start) + len(start)
- end_index = original_text.index(end, start_index)
- return original_text[start_index:end_index].strip()
+ start_index = text.index(start) + len(start)
+ end_index = text.index(end, start_index)
+ return text[start_index:end_index].strip()
def _str_to_url(link_str: str) -> yarl.URL:
@@ -340,15 +340,6 @@ def remove_trailing_slash(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
return url.parent.with_fragment(url.fragment).with_query(url.query)
-def remove_parts(
- url: AbsoluteHttpURL, *parts_to_remove: str, keep_query: bool = True, keep_fragment: bool = True
-) -> AbsoluteHttpURL:
- if not parts_to_remove:
- return url
- new_parts = [p for p in url.parts[1:] if p not in parts_to_remove]
- return url.with_path("/".join(new_parts), keep_fragment=keep_fragment, keep_query=keep_query)
-
-
@functools.cache
def get_system_information() -> dict[str, Any]:
def get_common_name() -> str: