Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: do not use get for expected values (#916)

NTFSvolumed18eed60fcc8
diff --git a/cyberdrop_dl/crawlers/bunkrr_albums_io.py b/cyberdrop_dl/crawlers/bunkrr_albums_io.py
index 1059a8ee7..9af74472f 100644
--- a/cyberdrop_dl/crawlers/bunkrr_albums_io.py
+++ b/cyberdrop_dl/crawlers/bunkrr_albums_io.py
@@ -31,15 +31,14 @@ class BunkrAlbumsIOCrawler(Crawler):
@create_task_id
async def fetch(self, scrape_item: ScrapeItem) -> None:
"""Determines where to send the scrape item based on the url."""
- if not scrape_item.url.query.get("search"): # Trying to scrape the root page is a bad idea
- raise ValueError
- await self.search(scrape_item)
+ if scrape_item.url.query.get("search"): # Trying to scrape the root page is a bad idea
+ return await self.search(scrape_item)
+ raise ValueError
@error_handling_wrapper
async def search(self, scrape_item: ScrapeItem) -> None:
"""Scrapes search results."""
- search_query = scrape_item.url.query.get("search")
- assert search_query
+ search_query = scrape_item.url.query["search"]
title = self.create_title(search_query)
scrape_item.setup_as_profile(title)
async for soup in self.web_pager(scrape_item):
@@ -50,7 +49,7 @@ class BunkrAlbumsIOCrawler(Crawler):
"""Generator of website pages."""
page_url = scrape_item.url
while True:
- current_page_number = int(page_url.query.get("page", 1))
+ current_page_number = int(page_url.query.get("page") or 1)
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.domain, page_url)
next_page = soup.select(self.next_page_selector)
@@ -59,6 +58,6 @@ class BunkrAlbumsIOCrawler(Crawler):
break
page_url_str: str = next_page[-1].get("href") # type: ignore
page_url = self.parse_url(page_url_str)
- next_page_number = int(page_url.query.get("page", 1))
+ next_page_number = int(page_url.query.get("page") or 1)
if current_page_number >= next_page_number:
break