Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: update selectors (Filester) (#83)
NTFSvolumeddea70c6044b
diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.pyindex 96ef9f8a7..4a6cb964a 100644--- a/cyberdrop_dl/crawlers/filester.py+++ b/cyberdrop_dl/crawlers/filester.py@@ -3,7 +3,7 @@ from __future__ import annotations #import randomfrom typing import TYPE_CHECKING, ClassVar-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css, open_graphfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between@@ -17,13 +17,15 @@ _CDN_URLS = AbsoluteHttpURL("https://cache1.filester.me"), AbsoluteHttpURL("httpclass Selector:FILES = ".file-item[onclick]"NEXT_PAGE = "a.page-link:-soup-contains(→)"+ FILE_DETAILS = "#detailsContent"@staticmethoddef file_attr(name: str) -> str:- return f"#detailsContent span:-soup-contains({name}) + span"+ return f"span:-soup-contains({name}) + span"MIME_TYPE = file_attr("Type")SHA_256 = file_attr("SHA-256")+ MD5 = file_attr("MD5")UPLOAD_DATE = file_attr("Uploaded")@@ -34,7 +36,7 @@ class FilesterCrawler(Crawler):}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://filester.me")DOMAIN: ClassVar[str] = "filester"- _RATE_LIMIT = 10, 1+ _RATE_LIMIT: ClassVar[RateLimit] = 3, 2async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:@@ -74,14 +76,20 @@ class FilesterCrawler(Crawler):returnsoup = await self.request_soup(scrape_item.url)- checksum = css.select_text(soup, Selector.SHA_256)- if await self.check_complete_by_hash(scrape_item, "sha256", checksum):+ file = css.select(soup, Selector.FILE_DETAILS)++ try:+ hash, checksum = "sha256", css.select_text(file, Selector.SHA_256)+ except css.SelectorError:+ hash, checksum = "md5", css.select_text(file, Selector.MD5)++ if await self.check_complete_by_hash(scrape_item, hash, checksum):returndl_link = await self._request_download(slug)name = open_graph.title(soup)- mime_type = css.select_text(soup, Selector.MIME_TYPE)- scrape_item.uploaded_at = self.parse_iso_date(css.select_text(soup, Selector.UPLOAD_DATE))+ mime_type = css.select_text(file, Selector.MIME_TYPE)+ scrape_item.uploaded_at = self.parse_iso_date(css.select_text(file, Selector.UPLOAD_DATE))filename, ext = self.get_filename_and_ext(name, mime_type=mime_type)await self.handle_file(scrape_item.url, scrape_item, name, ext, custom_filename=filename, debrid_link=dl_link)diff --git a/tests/crawlers/test_cases/filester.py b/tests/crawlers/test_cases/filester.pyindex fc6a3079e..84f6bbb3b 100644--- a/tests/crawlers/test_cases/filester.py+++ b/tests/crawlers/test_cases/filester.py@@ -4,7 +4,7 @@ TEST_CASES = ["https://filester.me/d/Y9Vkbpq",[{- "url": "re:https://cache1.filester.me/d/",+ "url": "re:.filester.me/d/","filename": "lilcanadiangirl_Stranger-in-the-Theatre.mp4","referer": "https://filester.me/d/Y9Vkbpq","album_id": None,@@ -12,11 +12,21 @@ TEST_CASES = [}],),+ (+ "https://filester.me/d/4h9lQtR",+ [+ {+ "url": "re:.filester.me/d/",+ "filename": "Linda Lan - Meanbitches slave orders.mp4",+ "album_id": None,+ }+ ],+ ),("https://filester.me/f/c3bf3e1da9982845",[{- "url": "re:https://cache1.filester.me/d/",+ "url": "re:.filester.me/d/","download_folder": "re:mirror_post-410_1771095844 (Filester)","album_id": "c3bf3e1da9982845",}