Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: split cyberfile and iceyfile crawlers (#917)

NTFSvolume030e387cb269
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 831af72e6..16ad84bde 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -29,6 +29,7 @@ from .generic import GenericCrawler
from .gofile import GoFileCrawler
from .google_drive import GoogleDriveCrawler
from .hotpic import HotPicCrawler
+from .iceyfile import IceyFileCrawler
from .imagebam import ImageBamCrawler
from .imgbb import ImgBBCrawler
from .imgbox import ImgBoxCrawler
diff --git a/cyberdrop_dl/crawlers/cyberfile.py b/cyberdrop_dl/crawlers/cyberfile.py
index b44a4d229..694d3c824 100644
--- a/cyberdrop_dl/crawlers/cyberfile.py
+++ b/cyberdrop_dl/crawlers/cyberfile.py
@@ -1,9 +1,7 @@
from __future__ import annotations
-import calendar
-import datetime
import itertools
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING
from aiolimiter import AsyncLimiter
from bs4 import BeautifulSoup
@@ -24,36 +22,32 @@ SOUP_ERRORS = {
}
-DOWNLOAD_BUTTON_SELECTOR = 'div[class="btn-group responsiveMobileMargin"] button'
-FILE_MENU_SELECTOR = 'ul[class="dropdown-menu dropdown-info account-dropdown-resize-menu"] li a'
-FILE_NAME_SELECTOR = "div.image-name-title"
-FILE_UPLOAD_DATE_SELECTOR = 'table[class="table table-bordered table-striped"] tr td[class=responsiveTable]'
-FOLDER_ID_JS_SELECTOR = "div[class*='page-container'] script:contains('loadImages')"
-FOLDER_ID_SELECTOR = "#folderId"
-FOLDER_ITEM_SELECTOR = "div[class=fileListing] div[class*=fileItem]"
-FOLDER_N_PAGES_SELECTOR = "a[onclick*=loadImages]"
-LOGIN_FORM_SELECTOR = "form[id=form_login]"
-PASSWORD_FORM_SELECTOR = "form[method='POST']"
-SHARED_N_PAGES_SELECTOR = "input[id=rspTotalPages]"
-SHOW_FILE_INFO_JS_SELECTOR = "script:contains('showFileInformation')"
+class Selectors:
+ DOWNLOAD_BUTTON = 'div[class="btn-group responsiveMobileMargin"] button'
+ FILE_MENU = 'ul[class="dropdown-menu dropdown-info account-dropdown-resize-menu"] li a'
+ FILE_NAME = "div.image-name-title"
+ FILE_UPLOAD_DATE = 'table[class="table table-bordered table-striped"] tr td[class=responsiveTable]'
+ FOLDER_ID_JS = "div[class*='page-container'] script:contains('loadImages')"
+ FOLDER_ID = "#folderId"
+ FOLDER_ITEM = "div[class=fileListing] div[class*=fileItem]"
+ FOLDER_N_PAGES = "a[onclick*=loadImages]"
+ LOGIN_FORM = "form[id=form_login]"
+ PASSWORD_FORM = "form[method='POST']"
+ SHARED_N_PAGES = "input[id=rspTotalPages]"
+ SHOW_FILE_INFO_JS = "script:contains('showFileInformation')"
+
+
+_SELECTOR = Selectors()
class CyberfileCrawler(Crawler):
- PRIMARY_BASE_DOMAINS: ClassVar[dict[str, URL]] = {
- "cyberfile": URL("https://cyberfile.me/"),
- "iceyfile": URL("https://iceyfile.com/"),
- }
-
- FOLDER_DOMAINS: ClassVar[dict[str, str]] = {"cyberfile": "Cyberfile", "iceyfile": "Iceyfile"}
- SUPPORTED_SITES: ClassVar[dict[str, list]] = {"cyberfile": ["cyberfile"], "iceyfile": ["iceyfile"]}
- update_unsupported = True
-
- def __init__(self, manager: Manager, site: str) -> None:
- super().__init__(manager, site, self.FOLDER_DOMAINS.get(site, "Cyberfile"))
- self.primary_base_domain = self.PRIMARY_BASE_DOMAINS.get(site, URL(f"https://{site}")) # type: ignore
- self.api_load_files = self.primary_base_domain / "account/ajax/load_files"
- self.api_details = self.primary_base_domain / "account/ajax/file_details"
- self.api_password_process = self.primary_base_domain / "ajax/folder_password_process"
+ primary_base_domain = URL("https://cyberfile.me/")
+
+ def __init__(self, manager: Manager) -> None:
+ super().__init__(manager, "cyberfile", "Cyberfile")
+ self.folders_api_url = self.primary_base_domain / "account/ajax/load_files"
+ self.files_api_url = self.primary_base_domain / "account/ajax/file_details"
+ self.folder_password_api_url = self.primary_base_domain / "ajax/folder_password_process"
self.request_limiter = AsyncLimiter(5, 1)
"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
@@ -73,23 +67,23 @@ class CyberfileCrawler(Crawler):
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)
- if soup.select_one(LOGIN_FORM_SELECTOR):
+ if soup.select_one(_SELECTOR.LOGIN_FORM):
raise ScrapeError(410, "Folder has been deleted")
- js_text = soup.select(FOLDER_ID_JS_SELECTOR)[-1].text
+ js_text = soup.select(_SELECTOR.FOLDER_ID_JS)[-1].text
# ex: loadImages('folder', '12345', 1, 0, '', {'searchTerm': "", 'filterUploadedDateRange': ""});
js_text = get_text_between(js_text, "loadImages(", ");")
- node_id = int(js_text.split(",")[1].replace("'", ""))
+ _, node_id = js_text.replace("'", "").split(",", 1)
album_id = scrape_item.url.parts[2]
title: str = ""
default_data = {"pageType": "folder", "perPage": 0, "filterOrderBy": ""}
for page in itertools.count(1):
data = default_data | {"nodeId": node_id, "pageStart": page}
ajax_soup, ajax_title = await self.get_soup_from_ajax(data, scrape_item)
- if page == 1:
+ if not title:
title = self.create_title(ajax_title, album_id)
scrape_item.setup_as_album(title, album_id=album_id)
- n_pages_text: str = ajax_soup.select(FOLDER_N_PAGES_SELECTOR)[-1]["onclick"] # type: ignore
+ n_pages_text: str = ajax_soup.select(_SELECTOR.FOLDER_N_PAGES)[-1]["onclick"] # type: ignore
n_pages = int(n_pages_text.split(",")[2].split(")")[0].strip())
_ = self.iter_files(scrape_item, ajax_soup)
@@ -113,7 +107,7 @@ class CyberfileCrawler(Crawler):
if page == 1:
title = self.create_title(ajax_title, album_id)
scrape_item.setup_as_album(title, album_id=album_id)
- n_pages = int(ajax_soup.select_one(SHARED_N_PAGES_SELECTOR)["value"]) # type: ignore
+ n_pages = int(ajax_soup.select_one(_SELECTOR.SHARED_N_PAGES)["value"]) # type: ignore
subfolders.extend(self.iter_files(scrape_item, ajax_soup, iter_subfolders=False))
page += 1
@@ -128,7 +122,7 @@ class CyberfileCrawler(Crawler):
"""Scrapes a file."""
def get_content_id(soup: BeautifulSoup) -> int | None:
- if file_info := soup.select_one(SHOW_FILE_INFO_JS_SELECTOR):
+ if file_info := soup.select_one(_SELECTOR.SHOW_FILE_INFO_JS):
content_id = get_text_between(file_info.text, "showFileInformation(", ");")
return int(content_id)
@@ -143,7 +137,7 @@ class CyberfileCrawler(Crawler):
soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)
if is_password_protected(soup):
- form = soup.select_one(PASSWORD_FORM_SELECTOR)
+ form = soup.select_one(_SELECTOR.PASSWORD_FORM)
if not form:
raise PasswordProtectedError("Unable to parse Password Protected File details")
@@ -169,7 +163,7 @@ class CyberfileCrawler(Crawler):
ajax_soup, page_title = await self.get_soup_from_ajax(data, scrape_item, is_file=True)
try:
- file_tag = ajax_soup.select_one(FILE_MENU_SELECTOR) or ajax_soup.select(DOWNLOAD_BUTTON_SELECTOR)[-1]
+ file_tag = ajax_soup.select_one(_SELECTOR.FILE_MENU) or ajax_soup.select(_SELECTOR.DOWNLOAD_BUTTON)[-1]
html_download_text = file_tag["onclick"]
link_str = html_download_text.split("'")[1].strip().removesuffix("'") # type: ignore
link = self.parse_url(link_str)
@@ -177,10 +171,10 @@ class CyberfileCrawler(Crawler):
check_soup_error(ajax_soup)
raise ScrapeError(422, "Couldn't find download button") from None
- if uploaded_date := ajax_soup.select_one(FILE_UPLOAD_DATE_SELECTOR):
- scrape_item.possible_datetime = parse_datetime(uploaded_date.text.strip())
+ if uploaded_date := ajax_soup.select_one(_SELECTOR.FILE_UPLOAD_DATE):
+ scrape_item.possible_datetime = self.parse_date(uploaded_date.text.strip(), "%d/%m/%Y %H:%M:%S")
- if ajax_title := ajax_soup.select_one(FILE_NAME_SELECTOR):
+ if ajax_title := ajax_soup.select_one(_SELECTOR.FILE_NAME):
filename = ajax_title.text
else:
filename = page_title
@@ -193,7 +187,7 @@ class CyberfileCrawler(Crawler):
Returns a list with the `folder_id` of every subfolder"""
folder_ids = []
- for item in soup.select(FOLDER_ITEM_SELECTOR):
+ for item in soup.select(_SELECTOR.FOLDER_ITEM):
folder_id, file_id = item["folderid"], item["fileid"]
if folder_id:
folder_ids.append(folder_ids)
@@ -225,13 +219,13 @@ class CyberfileCrawler(Crawler):
return BeautifulSoup(html.replace("\\", ""), "html.parser"), json_resp["page_title"]
password = scrape_item.url.query.get("password", "")
- ajax_url = self.api_details if is_file else self.api_load_files
+ ajax_url = self.files_api_url if is_file else self.folders_api_url
ajax_soup, ajax_title = await get_ajax_info()
if is_password_protected(ajax_soup):
if not password:
raise PasswordProtectedError
try:
- node_id = data.get("nodeId") or ajax_soup.select(FOLDER_ID_SELECTOR)[0]["value"]
+ node_id = data.get("nodeId") or ajax_soup.select(_SELECTOR.FOLDER_ID)[0]["value"]
except (IndexError, AttributeError):
check_soup_error(ajax_soup)
raise ScrapeError(422, message="nodeId not found") from None
@@ -239,7 +233,7 @@ class CyberfileCrawler(Crawler):
# Make a request with the password. Access to the file/folder will be stored in cookies
pw_data = {"folderPassword": password, "folderId": node_id, "submitme": 1}
async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.domain, self.api_password_process, data=pw_data)
+ json_resp: dict = await self.client.post_data(self.domain, self.folder_password_api_url, data=pw_data)
if not json_resp.get("success"):
raise PasswordProtectedError(message="Incorrect password")
@@ -248,12 +242,6 @@ class CyberfileCrawler(Crawler):
return ajax_soup, ajax_title
-def parse_datetime(date: str) -> int:
- """Parses a datetime string into a unix timestamp."""
- parsed_date = datetime.datetime.strptime(date, "%d/%m/%Y %H:%M:%S")
- return calendar.timegm(parsed_date.timetuple())
-
-
def is_password_protected(soup: BeautifulSoup) -> bool:
html = soup.get_text()
return any(text in html for text in ("Enter File Password", "Password Required"))
diff --git a/cyberdrop_dl/crawlers/iceyfile.py b/cyberdrop_dl/crawlers/iceyfile.py
new file mode 100644
index 000000000..1b308b51d
--- /dev/null
+++ b/cyberdrop_dl/crawlers/iceyfile.py
@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from yarl import URL
+
+from cyberdrop_dl.crawlers.cyberfile import CyberfileCrawler
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.managers.manager import Manager
+
+
+class IceyFileCrawler(CyberfileCrawler):
+ primary_base_domain = URL("https://iceyfile.com/")
+
+ def __init__(self, manager: Manager) -> None:
+ super().__init__(manager)
+ self.domain = "iceyfile"
+ self.folder_domain = "Iceyfile"