Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove redundant utils
NTFSvolume918f41840c4c
diff --git a/cyberdrop_dl/aio.py b/cyberdrop_dl/aio.pyindex ab15b729e..7b8f1ffc9 100644--- a/cyberdrop_dl/aio.py+++ b/cyberdrop_dl/aio.py@@ -8,30 +8,15 @@ import dataclassesimport shutilimport sysimport tempfile-from abc import ABC, abstractmethodfrom pathlib import Pathfrom stat import S_ISREG-from typing import (- IO,- TYPE_CHECKING,- Any,- AnyStr,- Generic,- ParamSpec,- Self,- TypeVar,- TypeVarTuple,- cast,- final,- overload,-)+from typing import IO, TYPE_CHECKING, Any, AnyStr, Generic, ParamSpec, Self, TypeVar, TypeVarTuple, cast, overloadfrom weakref import WeakValueDictionaryfrom typing_extensions import Sentinelif TYPE_CHECKING:from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterable, Iterator- from types import TracebackTypefrom _typeshed import OpenBinaryMode, OpenTextMode@@ -116,38 +101,6 @@ class AsyncIteratorWrapper(Generic[_T]):return cast("_T", value)-class AsyncContextManagerMixin(ABC):- __stack: contextlib.AsyncExitStack | None = None-- @final- @property- def _stack(self) -> contextlib.AsyncExitStack[bool | None]:- if self.__stack is None:- raise RuntimeError(f"{type(self).__name__} can only be used as a context manager")- return self.__stack-- @final- async def __aenter__(self) -> Self:- if self.__stack is not None:- raise RuntimeError(f"{type(self).__name__} does not allow reentrance")- self.__stack = contextlib.AsyncExitStack()- _ = await self.__stack.__aenter__()- await self._async_ctx_()- return self-- @final- async def __aexit__(- self,- exc_type: type[BaseException] | None,- exc_val: BaseException | None,- exc_tb: TracebackType | None,- ) -> bool | None:- return await self._stack.__aexit__(exc_type, exc_val, exc_tb)-- @abstractmethod- async def _async_ctx_(self) -> None: ...--async def gather(*coros: Awaitable[_T]) -> list[_T]:"""Like asyncio.gather but an exception on any coro cancels all pending corosdiff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex c109b1e2e..3b7048bcd 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -22,7 +22,7 @@ from bs4 import BeautifulSoup, Tagfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import LoginError, MaxChildrenError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, is_blob_or_svg+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, is_blob_or_svgfrom cyberdrop_dl.utils.dates import TimeStamp, to_timestampif TYPE_CHECKING:@@ -535,7 +535,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):def _lazy_load_embeds(self, post: ForumPostProtocol) -> Iterable[str]:selector = self.SELECTORS.posts.lazy_load_embedsfor lazy_media in css.iselect(post.content, selector.element):- yield get_text_between(css.attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")+ yield extr_text(css.attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:async for soup in self.web_pager(scrape_item.url, self.get_next_page):diff --git a/cyberdrop_dl/crawlers/_kvs.py b/cyberdrop_dl/crawlers/_kvs.pyindex 5b7538e54..016277670 100644--- a/cyberdrop_dl/crawlers/_kvs.py+++ b/cyberdrop_dl/crawlers/_kvs.py@@ -11,7 +11,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.exceptions import DownloadError, ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, open_graph, parse_url+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, open_graph, parse_urlif TYPE_CHECKING:from collections.abc import Sequence@@ -190,7 +190,7 @@ class KernelVideoSharingCrawler(Crawler, is_abc=True):soup = await self.request_soup(scrape_item.url)if not album_id:js_text = css.select_text(soup, Selector.ALBUM_ID)- album_id = get_text_between(js_text, "params['album_id'] =", ";")+ album_id = extr_text(js_text, "params['album_id'] =", ";")results = await self.get_album_results(album_id)title = css.select_text(soup, Selector.ALBUM_NAME)diff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.pyindex e99feae70..9a102661d 100644--- a/cyberdrop_dl/crawlers/_yetishare.py+++ b/cyberdrop_dl/crawlers/_yetishare.py@@ -7,7 +7,7 @@ from bs4 import BeautifulSoupfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_idfrom cyberdrop_dl.exceptions import DDOSGuardError, PasswordProtectedError, ScrapeError-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_textif TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem@@ -80,7 +80,7 @@ class YetiShareCrawler(Crawler, is_abc=True):else:# ex: loadImages('folder', '12345', 1, 0, '', {'searchTerm': "", 'filterUploadedDateRange': ""});page_type = "folder"- load_images = get_text_between(soup.select(Selector.LOAD_IMAGES)[-1].text, "loadImages(", ");")+ load_images = extr_text(soup.select(Selector.LOAD_IMAGES)[-1].text, "loadImages(", ");")node_id = load_images.replace("'", "").split(",")[1].strip()page = 1@@ -135,7 +135,7 @@ class YetiShareCrawler(Crawler, is_abc=True):_check_is_available(soup)content_id = int(- get_text_between(+ extr_text(css.select_text(soup, Selector.FILE_INFO),"showFileInformation(",");",@@ -156,7 +156,7 @@ class YetiShareCrawler(Crawler, is_abc=True):# Manually parse link. Some URLs are invalid. ex: https://cyberfile.me/7cfu# For the download URL, the slug does not actually matter. It can be anything- raw_link = get_text_between(css.attr(download_tag, "onclick"), "('", "');")+ raw_link = extr_text(css.attr(download_tag, "onclick"), "('", "');")token = raw_link.rpartition("?download_token=")[-1]link = self.parse_url(raw_link).with_query(download_token=token)diff --git a/cyberdrop_dl/crawlers/archivebate.py b/cyberdrop_dl/crawlers/archivebate.pyindex b3e6ff8f9..52e074029 100644--- a/cyberdrop_dl/crawlers/archivebate.py+++ b/cyberdrop_dl/crawlers/archivebate.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, RateLimitfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, open_graph+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, open_graphif TYPE_CHECKING:from cyberdrop_dl.crawlers.crawler import SupportedPaths@@ -44,7 +44,7 @@ class ArchiveBateCrawler(Crawler):if "This video has been deleted" in soup.get_text():raise ScrapeError(410)- upload_date = get_text_between(open_graph.description(soup), "show on", " - ").strip()+ upload_date = extr_text(open_graph.description(soup), "show on", " - ").strip()user_name = css.select_text(soup, Selector.USER_NAME)site_name = css.select_text(soup, Selector.SITE_NAME)scrape_item.setup_as_profile(self.create_title(f"{user_name} [{site_name}]"))diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.pyindex 2cbe7f426..bf11c939d 100644--- a/cyberdrop_dl/crawlers/ashemaletube.py+++ b/cyberdrop_dl/crawlers/ashemaletube.py@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, m3u8+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, m3u8if TYPE_CHECKING:from bs4 import BeautifulSoup, Tag@@ -151,7 +151,7 @@ class AShemaleTubeCrawler(Crawler):link_str: str = css.attr(image, "src")else:style: str = css.select(img_tag, "a", "style")- link_str = get_text_between(style, "url('", "');")+ link_str = extr_text(style, "url('", "');")url = self.parse_url(link_str).with_query(None)filename, ext = self.get_filename_and_ext(url.name)custom_filename = self.create_custom_filename(filename, ext, file_id=css.attr(img_tag, "data-image-id"))@@ -190,7 +190,7 @@ class AShemaleTubeCrawler(Crawler):)async def parse_player_info(self, script_text: str) -> tuple[Resolution, AbsoluteHttpURL, m3u8.Rendition]:- sources = get_text_between(script_text, "sources: ", "aspectRatio").strip().strip(",")+ sources = extr_text(script_text, "sources: ", "aspectRatio").strip().strip(",")sources_data = json.loads(sources)url = self.parse_url(sources_data["hlsAuto"])diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.pyindex 4d14e3f80..e9702006a 100644--- a/cyberdrop_dl/crawlers/doodstream.py+++ b/cyberdrop_dl/crawlers/doodstream.py@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_textif TYPE_CHECKING:from bs4 import BeautifulSoup@@ -96,7 +96,7 @@ class DoodStreamCrawler(Crawler):def _md5_pass(soup: BeautifulSoup) -> str:js_text = css.select_text(soup, Selector.MD5_JS)- return get_text_between(js_text, "/pass_md5/", "'")+ return extr_text(js_text, "/pass_md5/", "'")def _file_id(soup: BeautifulSoup) -> str:diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.pyindex fb0f49961..5102b98e0 100644--- a/cyberdrop_dl/crawlers/eporner.py+++ b/cyberdrop_dl/crawlers/eporner.py@@ -9,7 +9,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_textif TYPE_CHECKING:from bs4 import BeautifulSoup, Tag@@ -218,7 +218,7 @@ def _parse_video(soup: BeautifulSoup) -> Video:formats = [VideoSource.parse(tag) for tag in soup.select(Selector.FORMATS)]return Video(- title=get_text_between(ld_json, 'name": "', '",'),- date=get_text_between(ld_json, 'uploadDate": "', '"'),+ title=extr_text(ld_json, 'name": "', '",'),+ date=extr_text(ld_json, 'uploadDate": "', '"'),best_src=max(formats),)diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.pyindex 0b97e5ad4..62afb9225 100644--- a/cyberdrop_dl/crawlers/filester.py+++ b/cyberdrop_dl/crawlers/filester.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, open_graph+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, open_graphif TYPE_CHECKING:from collections.abc import AsyncGenerator@@ -54,7 +54,7 @@ class FilesterCrawler(Crawler):scrape_item.setup_as_album(title, album_id=album_id)for on_click in css.iselect(soup, Selector.FILES, "onclick"):- web_url = self.parse_url(get_text_between(on_click, "'", "'"))+ web_url = self.parse_url(extr_text(on_click, "'", "'"))new_scrape_item = scrape_item.create_child(web_url)self.create_task(self.run(new_scrape_item))scrape_item.add_children()diff --git a/cyberdrop_dl/crawlers/flickr.py b/cyberdrop_dl/crawlers/flickr.pyindex 28462ddb4..a96a36443 100644--- a/cyberdrop_dl/crawlers/flickr.py+++ b/cyberdrop_dl/crawlers/flickr.py@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import error_handling_context, error_handling_wrapper, get_text_between+from cyberdrop_dl.utils import error_handling_context, error_handling_wrapper, extr_textif TYPE_CHECKING:from collections.abc import AsyncGenerator@@ -106,7 +106,7 @@ class FlickrAPI:self._crawler.disable_on_error("Unable to get public API key"),):text = await self._crawler.request_text(prints)- self.api_key = get_text_between(text, 'flickr.api.site_key = "', '"')+ self.api_key = extr_text(text, 'flickr.api.site_key = "', '"')async def _request(self, method: str, **params: Any) -> dict[str, Any]:api_url = self.API_ENDPOINT.with_query(diff --git a/cyberdrop_dl/crawlers/fucking_fast.py b/cyberdrop_dl/crawlers/fucking_fast.pyindex 9d873025f..91ba83a8d 100644--- a/cyberdrop_dl/crawlers/fucking_fast.py+++ b/cyberdrop_dl/crawlers/fucking_fast.py@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_textif TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem@@ -34,7 +34,7 @@ class FuckingFastCrawler(Crawler):soup = await self.request_soup(scrape_item.url)name = css.page_title(soup)- dl_link = get_text_between(str(soup), 'window.open("https:', ")").strip('"')+ dl_link = extr_text(str(soup), 'window.open("https:', ")").strip('"')link = self.parse_url(f"https:{dl_link}")filename, ext = self.get_filename_and_ext(name)await self.handle_file(scrape_item.url, scrape_item, name, ext, debrid_link=link, custom_filename=filename)diff --git a/cyberdrop_dl/crawlers/gupload.py b/cyberdrop_dl/crawlers/gupload.pyindex c117b7add..e74b07e29 100644--- a/cyberdrop_dl/crawlers/gupload.py+++ b/cyberdrop_dl/crawlers/gupload.py@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Finalfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import error_handling_wrapper, get_text_between, xor_decrypt+from cyberdrop_dl.utils import error_handling_wrapper, extr_text, xor_decryptif TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem@@ -61,5 +61,5 @@ class GUploadCrawler(Crawler):async def _request_video_config(self, url: AbsoluteHttpURL) -> dict[str, Any]:html = await self.request_text(url, impersonate=True)- config_text = get_text_between(html, "var _cfg = ", "');").partition("('")[-1]+ config_text = extr_text(html, "var _cfg = ", "');").partition("('")[-1]return _decode_config(config_text)diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.pyindex 0e8aec83e..1071bb350 100644--- a/cyberdrop_dl/crawlers/imgur.py+++ b/cyberdrop_dl/crawlers/imgur.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_textif TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem@@ -41,7 +41,7 @@ class ImgurCrawler(Crawler):soup = await self.request_soup(self.PRIMARY_URL)js_src = css.select(soup, "script[src*='/desktop-assets/js/main']", "src")js_text = await self.request_text(self.parse_url(js_src))- self.client_id = get_text_between(js_text, 'apiClientId:"', '"')+ self.client_id = extr_text(js_text, 'apiClientId:"', '"')async def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.host == _IMAGE_CDN.host:diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.pyindex 14d883553..b81af0e8e 100644--- a/cyberdrop_dl/crawlers/kemono.py+++ b/cyberdrop_dl/crawlers/kemono.py@@ -17,7 +17,7 @@ from cyberdrop_dl.exceptions import NoExtensionError, ScrapeErrorfrom cyberdrop_dl.models import AliasModelfrom cyberdrop_dl.models.validators import falsy_as, falsy_as_nonefrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, remove_parts+from cyberdrop_dl.utils import css, error_handling_wrapperfrom cyberdrop_dl.utils.dates import to_timestampif TYPE_CHECKING:@@ -622,7 +622,7 @@ class KemonoCrawler(KemonoBaseCrawler):def _thumbnail_to_src(og_url: AbsoluteHttpURL) -> AbsoluteHttpURL:- url = remove_parts(og_url, "thumbnails", "thumbnail").with_query(None)+ url = og_url.with_path(og_url.path.replace("/thumbnails/", "/").replace("/thumbnail/", "/"))if name := og_url.query.get("f"):return url.with_query(f=name)return urldiff --git a/cyberdrop_dl/crawlers/leakedzone.py b/cyberdrop_dl/crawlers/leakedzone.pyindex db966b49f..83b5912b3 100644--- a/cyberdrop_dl/crawlers/leakedzone.py+++ b/cyberdrop_dl/crawlers/leakedzone.py@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.compat import IntEnumfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_textif TYPE_CHECKING:from bs4 import BeautifulSoup@@ -75,7 +75,7 @@ class LeakedZoneCrawler(Crawler):@classmethoddef get_encoded_video_url(cls, soup: BeautifulSoup) -> str:js_text = css.select_text(soup, _SELECTORS.JW_PLAYER)- return get_text_between(js_text, 'file: f("', '"),')+ return extr_text(js_text, 'file: f("', '"),')@error_handling_wrapperasync def model(self, scrape_item: ScrapeItem) -> None:diff --git a/cyberdrop_dl/crawlers/noodle_magazine.py b/cyberdrop_dl/crawlers/noodle_magazine.pyindex e20cfe79e..27bfe7adf 100644--- a/cyberdrop_dl/crawlers/noodle_magazine.py+++ b/cyberdrop_dl/crawlers/noodle_magazine.py@@ -9,7 +9,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, parse_url+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, parse_urlif TYPE_CHECKING:from collections.abc import Generator@@ -109,7 +109,7 @@ def _parse_video(soup: BeautifulSoup) -> Video:except css.SelectorError:raise ScrapeError(404) from None- playlist = json.loads(get_text_between(playlist_js, "window.playlist = ", ";\nwindow.ads"))+ playlist = json.loads(extr_text(playlist_js, "window.playlist = ", ";\nwindow.ads"))resolution, src = max(_parse_sources(playlist["sources"]))content_url = parse_url(json_ld["contentUrl"])diff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.pyindex e525ed9de..f3c114a7d 100644--- a/cyberdrop_dl/crawlers/odnoklassniki.py+++ b/cyberdrop_dl/crawlers/odnoklassniki.py@@ -7,7 +7,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, json+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, jsonif TYPE_CHECKING:from bs4 import BeautifulSoup@@ -80,7 +80,7 @@ class OdnoklassnikiCrawler(Crawler):soup = await self.request_soup(scrape_item.url, headers=_HEADERS)channel_id = channel_str.removeprefix("c")- gwt_hash = get_text_between(css.select_text(soup, Selector.CHANNEL_HASH), 'gwtHash:"', '",')+ gwt_hash = extr_text(css.select_text(soup, Selector.CHANNEL_HASH), 'gwtHash:"', '",')try:last_element_id = css.select(soup, *Selector.CHANNEL_LAST_ELEMENT)diff --git a/cyberdrop_dl/crawlers/patreon.py b/cyberdrop_dl/crawlers/patreon.pyindex de1e68e98..ed937a02b 100644--- a/cyberdrop_dl/crawlers/patreon.py+++ b/cyberdrop_dl/crawlers/patreon.py@@ -10,7 +10,7 @@ from typing_extensions import ReadOnlyfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import NoExtensionError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, next_js+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, next_jsif TYPE_CHECKING:from collections.abc import AsyncIterator, Generator@@ -214,7 +214,7 @@ class PatreonCrawler(Crawler):def _extract_campaign_id(soup: BeautifulSoup):- return get_text_between(str(soup), r"{\"value\":{\"campaign\":{\"data\":{\"id\":\"", r"\"")+ return extr_text(str(soup), r"{\"value\":{\"campaign\":{\"data\":{\"id\":\"", r"\"")def _extract_campaign_id_next_js(soup: BeautifulSoup) -> str:diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.pyindex 7d3a937dd..f5029008f 100644--- a/cyberdrop_dl/crawlers/pornhub.py+++ b/cyberdrop_dl/crawlers/pornhub.py@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal, NamedTuple, TypedDictfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_textif TYPE_CHECKING:from bs4 import BeautifulSoup@@ -301,12 +301,12 @@ class PornHubCrawler(Crawler):def get_upload_date_str(soup: BeautifulSoup) -> str:date_text = css.select_text(soup, _SELECTORS.DATE)- return get_text_between(date_text, 'uploadDate": "', '",')+ return extr_text(date_text, 'uploadDate": "', '",')def get_media_list(soup: BeautifulSoup) -> list[Media]:flashvars: str = css.select(soup, _SELECTORS.JS_VIDEO_INFO).text- media_text = get_text_between(flashvars, '"mediaDefinitions":', '"isVertical"').strip().removesuffix(",")+ media_text = extr_text(flashvars, '"mediaDefinitions":', '"isVertical"').strip().removesuffix(",")return json.loads(media_text)diff --git a/cyberdrop_dl/crawlers/sendvid.py b/cyberdrop_dl/crawlers/sendvid.pyindex 5a3066c09..e3ca5e32f 100644--- a/cyberdrop_dl/crawlers/sendvid.py+++ b/cyberdrop_dl/crawlers/sendvid.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, remove_parts+from cyberdrop_dl.utils import css, error_handling_wrapperif TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem@@ -67,7 +67,7 @@ class SendVidCrawler(Crawler):return PRIMARY_URL.with_path(video_id)if "embed" in url.parts:- return remove_parts(url, "embed")+ return url.with_path(url.path.replace("/embed/", "/"), keep_query=True, keep_fragment=True)return urldiff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.pyindex 8d1058b7e..50a717a49 100644--- a/cyberdrop_dl/crawlers/spankbang.py+++ b/cyberdrop_dl/crawlers/spankbang.py@@ -7,7 +7,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, json+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, jsonif TYPE_CHECKING:from collections.abc import Generator@@ -158,13 +158,13 @@ def _parse_video(soup: BeautifulSoup, display_id: str) -> Video:title_tag = css.select(soup, "div#video h1")stream_js_text = css.select_text(soup, Selector.STREAM_DATA)- stream_data = get_text_between(stream_js_text, "stream_data = ", ";")+ stream_data = extr_text(stream_js_text, "stream_data = ", ";")res, url = max(_parse_formats(stream_data))return Video(id=display_id,resolution=res,url=url,- stream_id=get_text_between(stream_js_text, "ana_video_id = ", ";").strip("'"),+ stream_id=extr_text(stream_js_text, "ana_video_id = ", ";").strip("'"),stream_key=css.select(soup, "[data-streamkey]", "data-streamkey"),title=css.attr_or_none(title_tag, "title") or css.text(title_tag),)diff --git a/cyberdrop_dl/crawlers/streamtape.py b/cyberdrop_dl/crawlers/streamtape.pyindex 9ee95d119..f3a9e048a 100644--- a/cyberdrop_dl/crawlers/streamtape.py+++ b/cyberdrop_dl/crawlers/streamtape.py@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, open_graph, parse_url+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, open_graph, parse_urlif TYPE_CHECKING:from bs4 import BeautifulSoup@@ -50,6 +50,6 @@ class StreamtapeCrawler(Crawler):def _extract_download_link(soup: BeautifulSoup) -> AbsoluteHttpURL:script = css.select_text(soup, Selector.JS_TOKEN)- token = get_text_between(script, "&token=", "'")+ token = extr_text(script, "&token=", "'")bait_url = css.select_text(soup, Selector.BAIT_LINK)return parse_url(f"https:/{bait_url}").update_query(token=token)diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex fc44e64b6..bb0c0130a 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -12,7 +12,7 @@ from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import error_handling_wrapper, get_text_between, parse_url, xor_decrypt+from cyberdrop_dl.utils import error_handling_wrapper, extr_text, parse_url, xor_decryptif TYPE_CHECKING:from collections.abc import Callable, Iterable@@ -238,7 +238,7 @@ class XhamsterCrawler(Crawler):async def _get_window_initials(self, url: AbsoluteHttpURL) -> dict[str, Any]:self._disable_ai_title_translations(url)content = await self.request_text(url)- initials = get_text_between(content, "window.initials=", ";</script>")+ initials = extr_text(content, "window.initials=", ";</script>")return json.loads(initials)diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.pyindex 6801f1dc7..5830c53a9 100644--- a/cyberdrop_dl/crawlers/xvideos.py+++ b/cyberdrop_dl/crawlers/xvideos.py@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_textif TYPE_CHECKING:from bs4 import BeautifulSoup@@ -120,7 +120,7 @@ class XVideosCrawler(Crawler):title = css.page_title(soup, self.DOMAIN)scrape_item.uploaded_at = self.parse_iso_date(css.json_ld(soup)["uploadDate"])script = css.select_text(soup, Selectors.HLS_VIDEO_JS)- m3u8_url = self.parse_url(get_text_between(script, "setVideoHLS('", "')"))+ m3u8_url = self.parse_url(extr_text(script, "setVideoHLS('", "')"))m3u8, info = await self.request_m3u8_playlist(m3u8_url)custom_filename = self.create_custom_filename(title, ".mp4", file_id=encoded_id, resolution=info.resolution)# Remove url slug to prevent duplicates in database. It's language specific and not required.@@ -135,14 +135,14 @@ class XVideosCrawler(Crawler):soup = await self._get_soup(scrape_item.url)script = css.select_text(soup, Selectors.ACCOUNT_INFO_JS)- display_name = get_text_between(script, '"display":"', '",')+ display_name = extr_text(script, '"display":"', '",')scrape_item.setup_as_profile(self.create_title(f"{display_name} [{name}]"))frag = scrape_item.url.fragmentpart = "photos" if "profiles" in scrape_item.url.parts else "post"if not frag or "_tabPhotos" in frag:galleries: dict[str, Any] | list[str] = json.loads(- get_text_between(script, '"galleries":', '"visitor":').removesuffix(",")+ extr_text(script, '"galleries":', '"visitor":').removesuffix(","))if isinstance(galleries, dict):@@ -158,11 +158,11 @@ class XVideosCrawler(Crawler):await self._iter_api_pages(scrape_item, scrape_item.url / "videos/new", "videos")if not frag or "quickies" in frag:- has_quickies = get_text_between(script, '"has_quickies":', ",").strip()+ has_quickies = extr_text(script, '"has_quickies":', ",").strip()if has_quickies == "false":return- user_id = get_text_between(script, '"id_user":', ",").strip()+ user_id = extr_text(script, '"id_user":', ",").strip()quickies_api = scrape_item.url.origin() / "quickies-api/profilevideos/all/none/N" / user_idawait self._iter_api_pages(scrape_item, quickies_api, "quickies")diff --git a/cyberdrop_dl/crawlers/youjizz.py b/cyberdrop_dl/crawlers/youjizz.pyindex 8ccee28ab..5964786a2 100644--- a/cyberdrop_dl/crawlers/youjizz.py+++ b/cyberdrop_dl/crawlers/youjizz.py@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, NamedTuplefrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import css, error_handling_wrapper, get_text_between, open_graph+from cyberdrop_dl.utils import css, error_handling_wrapper, extr_text, open_graphif TYPE_CHECKING:from collections.abc import Iterable@@ -76,7 +76,7 @@ class YouJizzCrawler(Crawler):def _parse_video(soup: BeautifulSoup) -> Video:js_text = css.select_text(soup, JS_SELECTOR)- encodings_text = get_text_between(js_text, "var dataEncodings =", "var encodings").strip().removesuffix(";")+ encodings_text = extr_text(js_text, "var dataEncodings =", "var encodings").strip().removesuffix(";")data_encodings = json.loads(encodings_text)return Video(title=open_graph.title(soup),diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.pyindex 3d3f8df2b..05164a7ef 100644--- a/cyberdrop_dl/utils/__init__.py+++ b/cyberdrop_dl/utils/__init__.py@@ -279,11 +279,11 @@ def _check_for_partial_files(path: Path) -> None:logger.warning("There are partial downloads in the downloads folder")-def get_text_between(original_text: str, start: str, end: str) -> str:+def extr_text(text: str, /, start: str, end: str) -> str:"""Extracts the text between two strings in a larger text. Result will be stripped"""- start_index = original_text.index(start) + len(start)- end_index = original_text.index(end, start_index)- return original_text[start_index:end_index].strip()+ start_index = text.index(start) + len(start)+ end_index = text.index(end, start_index)+ return text[start_index:end_index].strip()def _str_to_url(link_str: str) -> yarl.URL:@@ -340,15 +340,6 @@ def remove_trailing_slash(url: AbsoluteHttpURL) -> AbsoluteHttpURL:return url.parent.with_fragment(url.fragment).with_query(url.query)-def remove_parts(- url: AbsoluteHttpURL, *parts_to_remove: str, keep_query: bool = True, keep_fragment: bool = True-) -> AbsoluteHttpURL:- if not parts_to_remove:- return url- new_parts = [p for p in url.parts[1:] if p not in parts_to_remove]- return url.with_path("/".join(new_parts), keep_fragment=keep_fragment, keep_query=keep_query)--@functools.cachedef get_system_information() -> dict[str, Any]:def get_common_name() -> str: