Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: use new sintax (doodstream) (#918)
NTFSvolumeb99fdbb09f82
diff --git a/cyberdrop_dl/crawlers/doodstream.py b/cyberdrop_dl/crawlers/doodstream.pyindex fadefa7ee..0accc724c 100644--- a/cyberdrop_dl/crawlers/doodstream.py+++ b/cyberdrop_dl/crawlers/doodstream.py@@ -9,7 +9,7 @@ from yarl import URLfrom cyberdrop_dl.clients.errors import ScrapeErrorfrom cyberdrop_dl.crawlers.crawler import Crawler, create_task_id-from cyberdrop_dl.utils.utilities import error_handling_wrapper+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:from bs4 import BeautifulSoup@@ -18,17 +18,20 @@ if TYPE_CHECKING:from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem-VIDEO_SELECTOR = "div#video_player video"+class Selectors:+ VIDEO = "div#video_player video"+ MD5_JS = "script:contains('/pass_md5/')"+ FILE_ID_JS = "script:contains('file_id')"+++_SELECTORS = Selectors()API_MD5_ENTRYPOINT = URL("https://doodstream.com/pass_md5/")TOKEN_CHARS = string.ascii_letters + string.digits-JS_SELECTOR = "script:contains('/pass_md5/')"-FILE_JS_SELECTOR = "script:contains('file_id')"+DOOSTREAM_DOMAINS = ["vidply.com", "dood.re", "doodstream", "doodcdn", "doodstream.co", "dood.yt"]class DoodStreamCrawler(Crawler):- SUPPORTED_SITES: ClassVar[dict[str, list]] = {- "doodstream": ["vidply.com", "dood.re", "doodstream", "doodcdn", "doodstream.co", "dood.yt"]- }+ SUPPORTED_SITES: ClassVar[dict[str, list]] = {"doodstream": DOOSTREAM_DOMAINS}primary_base_domain = URL("https://doodstream.com/")update_unsupported = True@@ -72,7 +75,7 @@ class DoodStreamCrawler(Crawler):async with self.request_limiter:new_soup: BeautifulSoup = await self.client.get_soup_cffi(self.domain, api_url)- text = str(new_soup).strip()+ text = new_soup.get_text(strip=True)random_padding = "".join(random.choice(TOKEN_CHARS) for _ in range(10))expire = int(datetime.now(UTC).timestamp() * 1000) # remove decimalsdownload_url = self.parse_url(text + random_padding)@@ -80,19 +83,17 @@ class DoodStreamCrawler(Crawler):def get_md5_path(soup: BeautifulSoup) -> str:- js_info = soup.select_one(JS_SELECTOR)- js_text: str = js_info.text if js_info else ""+ js_text: str = js_info.text if (js_info := soup.select_one(_SELECTORS.MD5_JS)) else ""if not js_info:raise ScrapeError(422)- return js_text.split("/pass_md5/")[-1].split("'", 1)[0]+ return get_text_between(js_text, "/pass_md5/", "'")-def get_file_id(soup: BeautifulSoup) -> int:- js_info = soup.select_one(FILE_JS_SELECTOR)- js_text: str = js_info.text if js_info else ""+def get_file_id(soup: BeautifulSoup) -> str:+ js_text: str = js_info.text if (js_info := soup.select_one(_SELECTORS.FILE_ID_JS)) else ""if not js_info:raise ScrapeError(422)_, file_id, _ = js_text.split("'file_id'")[-1].split("'", 2)- return int(file_id)+ return file_id