Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: split cyberfile and iceyfile crawlers (#917)
NTFSvolume030e387cb269
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex 831af72e6..16ad84bde 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -29,6 +29,7 @@ from .generic import GenericCrawlerfrom .gofile import GoFileCrawlerfrom .google_drive import GoogleDriveCrawlerfrom .hotpic import HotPicCrawler+from .iceyfile import IceyFileCrawlerfrom .imagebam import ImageBamCrawlerfrom .imgbb import ImgBBCrawlerfrom .imgbox import ImgBoxCrawlerdiff --git a/cyberdrop_dl/crawlers/cyberfile.py b/cyberdrop_dl/crawlers/cyberfile.pyindex b44a4d229..694d3c824 100644--- a/cyberdrop_dl/crawlers/cyberfile.py+++ b/cyberdrop_dl/crawlers/cyberfile.py@@ -1,9 +1,7 @@from __future__ import annotations-import calendar-import datetimeimport itertools-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKINGfrom aiolimiter import AsyncLimiterfrom bs4 import BeautifulSoup@@ -24,36 +22,32 @@ SOUP_ERRORS = {}-DOWNLOAD_BUTTON_SELECTOR = 'div[class="btn-group responsiveMobileMargin"] button'-FILE_MENU_SELECTOR = 'ul[class="dropdown-menu dropdown-info account-dropdown-resize-menu"] li a'-FILE_NAME_SELECTOR = "div.image-name-title"-FILE_UPLOAD_DATE_SELECTOR = 'table[class="table table-bordered table-striped"] tr td[class=responsiveTable]'-FOLDER_ID_JS_SELECTOR = "div[class*='page-container'] script:contains('loadImages')"-FOLDER_ID_SELECTOR = "#folderId"-FOLDER_ITEM_SELECTOR = "div[class=fileListing] div[class*=fileItem]"-FOLDER_N_PAGES_SELECTOR = "a[onclick*=loadImages]"-LOGIN_FORM_SELECTOR = "form[id=form_login]"-PASSWORD_FORM_SELECTOR = "form[method='POST']"-SHARED_N_PAGES_SELECTOR = "input[id=rspTotalPages]"-SHOW_FILE_INFO_JS_SELECTOR = "script:contains('showFileInformation')"+class Selectors:+ DOWNLOAD_BUTTON = 'div[class="btn-group responsiveMobileMargin"] button'+ FILE_MENU = 'ul[class="dropdown-menu dropdown-info account-dropdown-resize-menu"] li a'+ FILE_NAME = "div.image-name-title"+ FILE_UPLOAD_DATE = 'table[class="table table-bordered table-striped"] tr td[class=responsiveTable]'+ FOLDER_ID_JS = "div[class*='page-container'] script:contains('loadImages')"+ FOLDER_ID = "#folderId"+ FOLDER_ITEM = "div[class=fileListing] div[class*=fileItem]"+ FOLDER_N_PAGES = "a[onclick*=loadImages]"+ LOGIN_FORM = "form[id=form_login]"+ PASSWORD_FORM = "form[method='POST']"+ SHARED_N_PAGES = "input[id=rspTotalPages]"+ SHOW_FILE_INFO_JS = "script:contains('showFileInformation')"+++_SELECTOR = Selectors()class CyberfileCrawler(Crawler):- PRIMARY_BASE_DOMAINS: ClassVar[dict[str, URL]] = {- "cyberfile": URL("https://cyberfile.me/"),- "iceyfile": URL("https://iceyfile.com/"),- }-- FOLDER_DOMAINS: ClassVar[dict[str, str]] = {"cyberfile": "Cyberfile", "iceyfile": "Iceyfile"}- SUPPORTED_SITES: ClassVar[dict[str, list]] = {"cyberfile": ["cyberfile"], "iceyfile": ["iceyfile"]}- update_unsupported = True-- def __init__(self, manager: Manager, site: str) -> None:- super().__init__(manager, site, self.FOLDER_DOMAINS.get(site, "Cyberfile"))- self.primary_base_domain = self.PRIMARY_BASE_DOMAINS.get(site, URL(f"https://{site}")) # type: ignore- self.api_load_files = self.primary_base_domain / "account/ajax/load_files"- self.api_details = self.primary_base_domain / "account/ajax/file_details"- self.api_password_process = self.primary_base_domain / "ajax/folder_password_process"+ primary_base_domain = URL("https://cyberfile.me/")++ def __init__(self, manager: Manager) -> None:+ super().__init__(manager, "cyberfile", "Cyberfile")+ self.folders_api_url = self.primary_base_domain / "account/ajax/load_files"+ self.files_api_url = self.primary_base_domain / "account/ajax/file_details"+ self.folder_password_api_url = self.primary_base_domain / "ajax/folder_password_process"self.request_limiter = AsyncLimiter(5, 1)"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""@@ -73,23 +67,23 @@ class CyberfileCrawler(Crawler):async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)- if soup.select_one(LOGIN_FORM_SELECTOR):+ if soup.select_one(_SELECTOR.LOGIN_FORM):raise ScrapeError(410, "Folder has been deleted")- js_text = soup.select(FOLDER_ID_JS_SELECTOR)[-1].text+ js_text = soup.select(_SELECTOR.FOLDER_ID_JS)[-1].text# ex: loadImages('folder', '12345', 1, 0, '', {'searchTerm': "", 'filterUploadedDateRange': ""});js_text = get_text_between(js_text, "loadImages(", ");")- node_id = int(js_text.split(",")[1].replace("'", ""))+ _, node_id = js_text.replace("'", "").split(",", 1)album_id = scrape_item.url.parts[2]title: str = ""default_data = {"pageType": "folder", "perPage": 0, "filterOrderBy": ""}for page in itertools.count(1):data = default_data | {"nodeId": node_id, "pageStart": page}ajax_soup, ajax_title = await self.get_soup_from_ajax(data, scrape_item)- if page == 1:+ if not title:title = self.create_title(ajax_title, album_id)scrape_item.setup_as_album(title, album_id=album_id)- n_pages_text: str = ajax_soup.select(FOLDER_N_PAGES_SELECTOR)[-1]["onclick"] # type: ignore+ n_pages_text: str = ajax_soup.select(_SELECTOR.FOLDER_N_PAGES)[-1]["onclick"] # type: ignoren_pages = int(n_pages_text.split(",")[2].split(")")[0].strip())_ = self.iter_files(scrape_item, ajax_soup)@@ -113,7 +107,7 @@ class CyberfileCrawler(Crawler):if page == 1:title = self.create_title(ajax_title, album_id)scrape_item.setup_as_album(title, album_id=album_id)- n_pages = int(ajax_soup.select_one(SHARED_N_PAGES_SELECTOR)["value"]) # type: ignore+ n_pages = int(ajax_soup.select_one(_SELECTOR.SHARED_N_PAGES)["value"]) # type: ignoresubfolders.extend(self.iter_files(scrape_item, ajax_soup, iter_subfolders=False))page += 1@@ -128,7 +122,7 @@ class CyberfileCrawler(Crawler):"""Scrapes a file."""def get_content_id(soup: BeautifulSoup) -> int | None:- if file_info := soup.select_one(SHOW_FILE_INFO_JS_SELECTOR):+ if file_info := soup.select_one(_SELECTOR.SHOW_FILE_INFO_JS):content_id = get_text_between(file_info.text, "showFileInformation(", ");")return int(content_id)@@ -143,7 +137,7 @@ class CyberfileCrawler(Crawler):soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)if is_password_protected(soup):- form = soup.select_one(PASSWORD_FORM_SELECTOR)+ form = soup.select_one(_SELECTOR.PASSWORD_FORM)if not form:raise PasswordProtectedError("Unable to parse Password Protected File details")@@ -169,7 +163,7 @@ class CyberfileCrawler(Crawler):ajax_soup, page_title = await self.get_soup_from_ajax(data, scrape_item, is_file=True)try:- file_tag = ajax_soup.select_one(FILE_MENU_SELECTOR) or ajax_soup.select(DOWNLOAD_BUTTON_SELECTOR)[-1]+ file_tag = ajax_soup.select_one(_SELECTOR.FILE_MENU) or ajax_soup.select(_SELECTOR.DOWNLOAD_BUTTON)[-1]html_download_text = file_tag["onclick"]link_str = html_download_text.split("'")[1].strip().removesuffix("'") # type: ignorelink = self.parse_url(link_str)@@ -177,10 +171,10 @@ class CyberfileCrawler(Crawler):check_soup_error(ajax_soup)raise ScrapeError(422, "Couldn't find download button") from None- if uploaded_date := ajax_soup.select_one(FILE_UPLOAD_DATE_SELECTOR):- scrape_item.possible_datetime = parse_datetime(uploaded_date.text.strip())+ if uploaded_date := ajax_soup.select_one(_SELECTOR.FILE_UPLOAD_DATE):+ scrape_item.possible_datetime = self.parse_date(uploaded_date.text.strip(), "%d/%m/%Y %H:%M:%S")- if ajax_title := ajax_soup.select_one(FILE_NAME_SELECTOR):+ if ajax_title := ajax_soup.select_one(_SELECTOR.FILE_NAME):filename = ajax_title.textelse:filename = page_title@@ -193,7 +187,7 @@ class CyberfileCrawler(Crawler):Returns a list with the `folder_id` of every subfolder"""folder_ids = []- for item in soup.select(FOLDER_ITEM_SELECTOR):+ for item in soup.select(_SELECTOR.FOLDER_ITEM):folder_id, file_id = item["folderid"], item["fileid"]if folder_id:folder_ids.append(folder_ids)@@ -225,13 +219,13 @@ class CyberfileCrawler(Crawler):return BeautifulSoup(html.replace("\\", ""), "html.parser"), json_resp["page_title"]password = scrape_item.url.query.get("password", "")- ajax_url = self.api_details if is_file else self.api_load_files+ ajax_url = self.files_api_url if is_file else self.folders_api_urlajax_soup, ajax_title = await get_ajax_info()if is_password_protected(ajax_soup):if not password:raise PasswordProtectedErrortry:- node_id = data.get("nodeId") or ajax_soup.select(FOLDER_ID_SELECTOR)[0]["value"]+ node_id = data.get("nodeId") or ajax_soup.select(_SELECTOR.FOLDER_ID)[0]["value"]except (IndexError, AttributeError):check_soup_error(ajax_soup)raise ScrapeError(422, message="nodeId not found") from None@@ -239,7 +233,7 @@ class CyberfileCrawler(Crawler):# Make a request with the password. Access to the file/folder will be stored in cookiespw_data = {"folderPassword": password, "folderId": node_id, "submitme": 1}async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.domain, self.api_password_process, data=pw_data)+ json_resp: dict = await self.client.post_data(self.domain, self.folder_password_api_url, data=pw_data)if not json_resp.get("success"):raise PasswordProtectedError(message="Incorrect password")@@ -248,12 +242,6 @@ class CyberfileCrawler(Crawler):return ajax_soup, ajax_title-def parse_datetime(date: str) -> int:- """Parses a datetime string into a unix timestamp."""- parsed_date = datetime.datetime.strptime(date, "%d/%m/%Y %H:%M:%S")- return calendar.timegm(parsed_date.timetuple())--def is_password_protected(soup: BeautifulSoup) -> bool:html = soup.get_text()return any(text in html for text in ("Enter File Password", "Password Required"))diff --git a/cyberdrop_dl/crawlers/iceyfile.py b/cyberdrop_dl/crawlers/iceyfile.pynew file mode 100644index 000000000..1b308b51d--- /dev/null+++ b/cyberdrop_dl/crawlers/iceyfile.py@@ -0,0 +1,19 @@+from __future__ import annotations++from typing import TYPE_CHECKING++from yarl import URL++from cyberdrop_dl.crawlers.cyberfile import CyberfileCrawler++if TYPE_CHECKING:+ from cyberdrop_dl.managers.manager import Manager+++class IceyFileCrawler(CyberfileCrawler):+ primary_base_domain = URL("https://iceyfile.com/")++ def __init__(self, manager: Manager) -> None:+ super().__init__(manager)+ self.domain = "iceyfile"+ self.folder_domain = "Iceyfile"