Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: support subfolders (Filester) (#91)
NTFSvolume8eeff99897e4
diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.pyindex f6a765fa7..d3aa5bfe8 100644--- a/cyberdrop_dl/crawlers/filester.py+++ b/cyberdrop_dl/crawlers/filester.py@@ -9,6 +9,10 @@ from cyberdrop_dl.utils import css, open_graphfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:+ from collections.abc import AsyncGenerator++ from bs4 import BeautifulSoup+from cyberdrop_dl.data_structures.url_objects import ScrapeItem_CDN_URLS = AbsoluteHttpURL("https://cache1.filester.me"), AbsoluteHttpURL("https://cache6.filester.me")@@ -16,6 +20,7 @@ _CDN_URLS = AbsoluteHttpURL("https://cache1.filester.me"), AbsoluteHttpURL("httpclass Selector:FILES = ".file-item[onclick]"+ SUBFOLDER = ".subfolder-item[href]"NEXT_PAGE = "a.page-link:-soup-contains(→)"FILE_DETAILS = "#detailsContent"@@ -27,7 +32,7 @@ class FilesterCrawler(Crawler):}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://filester.me")DOMAIN: ClassVar[str] = "filester"- _RATE_LIMIT: ClassVar[RateLimit] = 3, 2+ _RATE_LIMIT: ClassVar[RateLimit] = 4, 1async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:@@ -40,26 +45,39 @@ class FilesterCrawler(Crawler):@error_handling_wrapperasync def folder(self, scrape_item: ScrapeItem, album_id: str) -> None:- soup = await self.request_soup(scrape_item.url)- name = open_graph.title(soup)- title = self.create_title(name, album_id)- scrape_item.setup_as_album(title, album_id=album_id)+ title: str = ""+ subfolders: list[str] = []- while True:- for row in soup.select(Selector.FILES):- url = get_text_between(css.attr(row, "onclick"), "'", "'")- web_url = self.parse_url(url)+ async for soup in self._folder_pager(scrape_item.url):+ if not title:+ name = open_graph.title(soup)+ title = self.create_title(name, album_id)+ scrape_item.setup_as_album(title, album_id=album_id)++ for on_click in css.iselect(soup, Selector.FILES, "onclick"):+ web_url = self.parse_url(get_text_between(on_click, "'", "'"))new_scrape_item = scrape_item.create_child(web_url)self.create_task(self.run(new_scrape_item))scrape_item.add_children()+ subfolders.extend(css.iselect(soup, Selector.SUBFOLDER, "href"))++ for subfolder in dict.fromkeys(subfolders):+ new_scrape_item = scrape_item.create_child(self.parse_url(subfolder))+ self.create_task(self.run(new_scrape_item))+ scrape_item.add_children()++ async def _folder_pager(self, url: AbsoluteHttpURL) -> AsyncGenerator[BeautifulSoup]:+ next_page = url+ while True:+ soup = await self.request_soup(next_page)+ yield souptry:query = css.select(soup, Selector.NEXT_PAGE, "href")except css.SelectorError:break- next_page = scrape_item.url.with_query(query.strip("?"))- soup = await self.request_soup(next_page)+ next_page = next_page.with_query(query.strip("?"))@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem, slug: str) -> None:diff --git a/tests/crawlers/test_cases/filester.py b/tests/crawlers/test_cases/filester.pyindex 84f6bbb3b..9a8ad6672 100644--- a/tests/crawlers/test_cases/filester.py+++ b/tests/crawlers/test_cases/filester.py@@ -33,4 +33,14 @@ TEST_CASES = [],53,),+ (+ "https://filester.me/f/c1a31c9ca510870b",+ [+ {+ "url": "re:.filester.me/d/",+ "download_folder": "re:Natalie (Filester)/Pics",+ }+ ],+ range(150, 160),+ ),]