Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: support subfolders (Filester) (#91)

NTFSvolume8eeff99897e4
diff --git a/cyberdrop_dl/crawlers/filester.py b/cyberdrop_dl/crawlers/filester.py
index f6a765fa7..d3aa5bfe8 100644
--- a/cyberdrop_dl/crawlers/filester.py
+++ b/cyberdrop_dl/crawlers/filester.py
@@ -9,6 +9,10 @@ from cyberdrop_dl.utils import css, open_graph
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator
+
+ from bs4 import BeautifulSoup
+
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
_CDN_URLS = AbsoluteHttpURL("https://cache1.filester.me"), AbsoluteHttpURL("https://cache6.filester.me")
@@ -16,6 +20,7 @@ _CDN_URLS = AbsoluteHttpURL("https://cache1.filester.me"), AbsoluteHttpURL("http
class Selector:
FILES = ".file-item[onclick]"
+ SUBFOLDER = ".subfolder-item[href]"
NEXT_PAGE = "a.page-link:-soup-contains(→)"
FILE_DETAILS = "#detailsContent"
@@ -27,7 +32,7 @@ class FilesterCrawler(Crawler):
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://filester.me")
DOMAIN: ClassVar[str] = "filester"
- _RATE_LIMIT: ClassVar[RateLimit] = 3, 2
+ _RATE_LIMIT: ClassVar[RateLimit] = 4, 1
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
@@ -40,26 +45,39 @@ class FilesterCrawler(Crawler):
@error_handling_wrapper
async def folder(self, scrape_item: ScrapeItem, album_id: str) -> None:
- soup = await self.request_soup(scrape_item.url)
- name = open_graph.title(soup)
- title = self.create_title(name, album_id)
- scrape_item.setup_as_album(title, album_id=album_id)
+ title: str = ""
+ subfolders: list[str] = []
- while True:
- for row in soup.select(Selector.FILES):
- url = get_text_between(css.attr(row, "onclick"), "'", "'")
- web_url = self.parse_url(url)
+ async for soup in self._folder_pager(scrape_item.url):
+ if not title:
+ name = open_graph.title(soup)
+ title = self.create_title(name, album_id)
+ scrape_item.setup_as_album(title, album_id=album_id)
+
+ for on_click in css.iselect(soup, Selector.FILES, "onclick"):
+ web_url = self.parse_url(get_text_between(on_click, "'", "'"))
new_scrape_item = scrape_item.create_child(web_url)
self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
+ subfolders.extend(css.iselect(soup, Selector.SUBFOLDER, "href"))
+
+ for subfolder in dict.fromkeys(subfolders):
+ new_scrape_item = scrape_item.create_child(self.parse_url(subfolder))
+ self.create_task(self.run(new_scrape_item))
+ scrape_item.add_children()
+
+ async def _folder_pager(self, url: AbsoluteHttpURL) -> AsyncGenerator[BeautifulSoup]:
+ next_page = url
+ while True:
+ soup = await self.request_soup(next_page)
+ yield soup
try:
query = css.select(soup, Selector.NEXT_PAGE, "href")
except css.SelectorError:
break
- next_page = scrape_item.url.with_query(query.strip("?"))
- soup = await self.request_soup(next_page)
+ next_page = next_page.with_query(query.strip("?"))
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem, slug: str) -> None:
diff --git a/tests/crawlers/test_cases/filester.py b/tests/crawlers/test_cases/filester.py
index 84f6bbb3b..9a8ad6672 100644
--- a/tests/crawlers/test_cases/filester.py
+++ b/tests/crawlers/test_cases/filester.py
@@ -33,4 +33,14 @@ TEST_CASES = [
],
53,
),
+ (
+ "https://filester.me/f/c1a31c9ca510870b",
+ [
+ {
+ "url": "re:.filester.me/d/",
+ "download_folder": "re:Natalie (Filester)/Pics",
+ }
+ ],
+ range(150, 160),
+ ),
]