Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: nested shared folders (yetishare) (#1242)

NTFSvolumebbf6bf7e0ba7
diff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.py
index 666786184..18f5dd46b 100644
--- a/cyberdrop_dl/crawlers/_yetishare.py
+++ b/cyberdrop_dl/crawlers/_yetishare.py
@@ -1,13 +1,13 @@
from __future__ import annotations
import itertools
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
from aiolimiter import AsyncLimiter
from bs4 import BeautifulSoup
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
-from cyberdrop_dl.exceptions import PasswordProtectedError, ScrapeError
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id
+from cyberdrop_dl.exceptions import DDOSGuardError, PasswordProtectedError, ScrapeError
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
@@ -15,237 +15,237 @@ if TYPE_CHECKING:
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-SOUP_ERRORS = {
- 410: ["File has been removed"],
- 401: ["File is not publicly available"],
-}
-
-
-class Selectors:
- DOWNLOAD_BUTTON = 'div[class="btn-group responsiveMobileMargin"] button'
- FILE_MENU = 'ul[class="dropdown-menu dropdown-info account-dropdown-resize-menu"] li a'
- FILE_NAME = "div.image-name-title"
+class Selector:
+ DOWNLOAD_BUTTON = ".btn-group.responsiveMobileMargin button:contains('Download')[onclick*='download_token']"
+ DROPDOWN_MENU = ".dropdown-menu.dropdown-info a[onclick*='download_token']"
+ FILE_NAME = ".image-name-title"
FILE_UPLOAD_DATE = "td:contains('Uploaded:') + td"
- FOLDER_ID_JS = "div[class*='page-container'] script:contains('loadImages')"
- FOLDER_ID = "#folderId"
- FOLDER_ITEM = "div[class=fileListing] div[class*=fileItem]"
- FOLDER_N_PAGES = "a[onclick*=loadImages]"
- LOGIN_FORM = "form[id=form_login]"
- PASSWORD_FORM = "form[method='POST']"
- SHARED_N_PAGES = "input[id=rspTotalPages]"
- SHOW_FILE_INFO_JS = "script:contains('showFileInformation')"
+ FILE_INFO = "script:contains('showFileInformation')"
+ LOAD_IMAGES = "div[class*='page-container'] script:contains('loadImages')"
+ FOLDER_ID = "#folderId"
+ _FOLDER_ITEM = "#fileListing [class*=fileItem]"
+ FILES = f"{_FOLDER_ITEM}[fileid]"
+ SUBFOLDERS = f"{_FOLDER_ITEM}[folderid]"
+ FOLDER_TOTAL_PAGES = "input#rspTotalPages"
-_SELECTOR = Selectors()
+ LOGIN_FORM = "form#form_login"
+ PASSWORD_PROTECTED = "#folderPasswordForm, #filePassword"
+ RECAPTCHA = "form[method=POST] script[src*='/recaptcha/api.js']"
class YetiShareCrawler(Crawler, is_abc=True):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Files": "/...",
- "Folder": "/folder/...",
- "Shared folder": "/shared/...",
+ "Files": (
+ "/<file_id>",
+ "/<file_id>/<file_name>",
+ ),
+ "Public Folders": (
+ "/folder/<folder_id>",
+ "/folder/<folder_id>/<folder_name>",
+ ),
+ "Shared folders": "/shared/<share_key>",
}
def __init_subclass__(cls, **kwargs) -> None:
super().__init_subclass__(**kwargs)
- cls.folders_api_url = cls.PRIMARY_URL / "account/ajax/load_files"
- cls.files_api_url = cls.PRIMARY_URL / "account/ajax/file_details"
- cls.folder_password_api_url = cls.PRIMARY_URL / "ajax/folder_password_process"
+ cls.FOLDERS_API_URL = cls.PRIMARY_URL / "account/ajax/load_files"
+ cls.FILE_API_URL = cls.PRIMARY_URL / "account/ajax/file_details"
+ cls.FOLDER_PASSWORD_API_URL = cls.PRIMARY_URL / "ajax/folder_password_process"
def __post_init__(self) -> None:
self.request_limiter = AsyncLimiter(5, 1)
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if "folder" in scrape_item.url.parts:
- return await self.folder(scrape_item)
- if "shared" in scrape_item.url.parts:
- return await self.shared_folder(scrape_item)
- return await self.file(scrape_item)
+ match scrape_item.url.parts[1:]:
+ case ["folder", folder_id, *_]:
+ return await self.folder(scrape_item, folder_id)
+ case ["shared", folder_id]:
+ return await self.folder(scrape_item, folder_id, is_shared=True)
+ case [file_id]:
+ return await self.file(scrape_item, file_id)
+ case [file_id, _]:
+ return await self.file(scrape_item, file_id)
+ case _:
+ raise ValueError
@error_handling_wrapper
- async def folder(self, scrape_item: ScrapeItem) -> None:
+ async def folder(self, scrape_item: ScrapeItem, folder_id: str, is_shared: bool = False) -> None:
+ # Make request to update cookies. Access to folders in saved on cookies
async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- if soup.select_one(_SELECTOR.LOGIN_FORM):
+ if soup.select_one(Selector.LOGIN_FORM):
raise ScrapeError(410, "Folder has been deleted")
- js_text = soup.select(_SELECTOR.FOLDER_ID_JS)[-1].text
- # ex: loadImages('folder', '12345', 1, 0, '', {'searchTerm': "", 'filterUploadedDateRange': ""});
- js_text = get_text_between(js_text, "loadImages(", ");")
- _, node_id = js_text.replace("'", "").split(",", 1)
- album_id = scrape_item.url.parts[2]
- title: str = ""
- default_data = {"pageType": "folder", "perPage": 0, "filterOrderBy": ""}
- n_pages = 1
- for page in itertools.count(1):
- data = default_data | {"nodeId": node_id, "pageStart": page}
- ajax_soup, ajax_title = await self.get_soup_from_ajax(data, scrape_item)
- if not title:
- title = self.create_title(ajax_title, album_id)
- scrape_item.setup_as_album(title, album_id=album_id)
- next_page_tag = ajax_soup.select(_SELECTOR.FOLDER_N_PAGES)[-1]
- n_pages_text: str = css.get_attr(next_page_tag, "onclick")
- n_pages = int(n_pages_text.split(",")[2].split(")")[0].strip())
-
- _ = self.iter_files(scrape_item, ajax_soup)
- if page >= n_pages:
- break
+ if is_shared:
+ page_type = "nonaccountshared"
+ node_id = ""
- @error_handling_wrapper
- async def shared_folder(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ else:
+ # ex: loadImages('folder', '12345', 1, 0, '', {'searchTerm': "", 'filterUploadedDateRange': ""});
+ page_type = "folder"
+ load_images = get_text_between(soup.select(Selector.LOAD_IMAGES)[-1].text, "loadImages(", ");")
+ node_id = load_images.replace("'", "").split(",")[1].strip()
- subfolders = []
- node_id = ""
- album_id = scrape_item.url.parts[2]
page = 1
- default_data = {"pageType": "nonaccountshared", "perPage": 0, "filterOrderBy": ""}
- n_pages = 1
- while True:
- data = default_data | {"nodeId": node_id, "pageStart": page}
- ajax_soup, ajax_title = await self.get_soup_from_ajax(data, scrape_item)
- if page == 1:
- title = self.create_title(ajax_title, album_id)
- scrape_item.setup_as_album(title, album_id=album_id)
- n_pages = int(css.select_one_get_attr(ajax_soup, _SELECTOR.SHARED_N_PAGES, "value"))
-
- subfolders.extend(self.iter_files(scrape_item, ajax_soup, iter_subfolders=False))
- page += 1
- if page > n_pages:
- if not subfolders:
- break
- node_id = str(subfolders.pop(0))
- page = 1
+ total_pages = None
+ if not scrape_item.album_id:
+ scrape_item.setup_as_album("", album_id=folder_id)
+
+ if not is_shared:
+ title = css.page_title(soup, self.DOMAIN).removesuffix("Folder").strip()
+ scrape_item.add_to_parent_title(self.create_title(title, folder_id))
+
+ for page in itertools.count(1):
+ ajax_soup = await self._get_soup_from_ajax_api(
+ scrape_item,
+ data={
+ "pageType": page_type,
+ "perPage": 0,
+ "filterOrderBy": "",
+ "nodeId": node_id,
+ "pageStart": page,
+ },
+ )
+
+ if total_pages is None:
+ total_pages = int(css.select_one_get_attr(ajax_soup, Selector.FOLDER_TOTAL_PAGES, "value"))
+
+ for file in ajax_soup.select(Selector.FILES):
+ file_url = self.parse_url(css.get_attr(file, "dtfullurl"))
+ content_id = css.get_attr(file, "fileid")
+ new_scrape_item = scrape_item.create_child(file_url)
+ self.create_task(self._handle_content_id_task(new_scrape_item, content_id))
+ scrape_item.add_children()
+
+ for _, new_scrape_item in self.iter_children(
+ scrape_item, ajax_soup, Selector.SUBFOLDERS, attribute="sharing-url"
+ ):
+ self.create_task(self.run(new_scrape_item))
+
+ if page >= total_pages:
+ break
@error_handling_wrapper
- async def file(self, scrape_item: ScrapeItem) -> None:
- def get_content_id(soup: BeautifulSoup) -> int | None:
- if file_info := soup.select_one(_SELECTOR.SHOW_FILE_INFO_JS):
- content_id = get_text_between(file_info.text, "showFileInformation(", ");")
- return int(content_id)
-
- file_id = scrape_item.url.parts[1]
- canonical_url = self.PRIMARY_URL / file_id
- if await self.check_complete_from_referer(canonical_url):
+ async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:
+ if await self.check_complete_from_referer(scrape_item):
return
- password = scrape_item.url.query.get("password", "")
- scrape_item.url = canonical_url
async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
- if is_password_protected(soup):
- form = soup.select_one(_SELECTOR.PASSWORD_FORM)
- if not form:
- raise PasswordProtectedError("Unable to parse Password Protected File details")
+ if soup.select_one(Selector.PASSWORD_PROTECTED):
+ soup = await self._unlock_password_protected_file(scrape_item, file_id)
- password_post_url = self.parse_url(css.get_attr(form, "action"))
- data = {"filePassword": password, "submitme": 1}
- async with self.request_limiter:
- resp_bytes = await self.client.post_data_raw(self.DOMAIN, password_post_url, data=data)
-
- soup = BeautifulSoup(resp_bytes, "html.parser")
- if is_password_protected(soup):
- raise PasswordProtectedError("File password is invalid")
+ _check_is_available(soup)
- if content_id := get_content_id(soup):
- return await self.handle_content_id(scrape_item, content_id)
+ content_id = int(
+ get_text_between(
+ css.select_one_get_text(soup, Selector.FILE_INFO),
+ "showFileInformation(",
+ ");",
+ )
+ )
- check_soup_error(soup)
- raise ScrapeError(422, message="contentId not found")
+ return await self._handle_content_id(scrape_item, content_id)
@error_handling_wrapper
- async def handle_content_id(self, scrape_item: ScrapeItem, content_id: int) -> None:
- data = {"u": content_id}
- ajax_soup, page_title = await self.get_soup_from_ajax(data, scrape_item, is_file=True)
-
- try:
- file_tag = ajax_soup.select_one(_SELECTOR.FILE_MENU) or ajax_soup.select(_SELECTOR.DOWNLOAD_BUTTON)[-1]
- html_download_text = css.get_attr(file_tag, "onclick")
- link_str = html_download_text.split("'")[1].strip().removesuffix("'")
- link = self.parse_url(link_str)
- except (AttributeError, IndexError, KeyError):
- check_soup_error(ajax_soup)
- raise ScrapeError(422, "Couldn't find download button") from None
-
- if uploaded_date := ajax_soup.select_one(_SELECTOR.FILE_UPLOAD_DATE):
- scrape_item.possible_datetime = self.parse_date(uploaded_date.text.strip(), "%d/%m/%Y %H:%M:%S")
-
- if ajax_title := ajax_soup.select_one(_SELECTOR.FILE_NAME):
- filename = ajax_title.get_text(strip=True)
- else:
- filename = page_title.strip()
-
- filename, ext = self.get_filename_and_ext(filename or link.name)
- await self.handle_file(link, scrape_item, filename, ext)
-
- def iter_files(self, scrape_item: ScrapeItem, soup: BeautifulSoup, *, iter_subfolders: bool = True) -> list[str]:
- """Proccess all the files in this folder. Optionally process subfolders
-
- Returns a list with the `folder_id` of every subfolder"""
- folder_ids = []
- for item in soup.select(_SELECTOR.FOLDER_ITEM):
- if folder_id := css.get_attr_or_none(item, "folderid"):
- folder_ids.append(folder_id)
- if not iter_subfolders:
- continue
- link_str = css.get_attr(item, "sharing-url")
- elif css.get_attr_or_none(item, "fileid"):
- link_str = css.get_attr(item, "dtfullurl")
- else:
- continue
-
- link = self.parse_url(link_str)
- new_scrape_item = scrape_item.create_child(link)
- self.manager.task_group.create_task(self.run(new_scrape_item))
- scrape_item.add_children()
- return folder_ids
-
- async def get_soup_from_ajax(
- self, data: dict, scrape_item: ScrapeItem, is_file: bool = False
- ) -> tuple[BeautifulSoup, str]:
- """Returns soup and page title as a tuple"""
-
- async def get_ajax_info() -> tuple[BeautifulSoup, str]:
+ async def _handle_content_id(self, scrape_item: ScrapeItem, content_id: int) -> None:
+ soup = await self._get_soup_from_ajax_api(
+ scrape_item,
+ data={"u": content_id},
+ is_file=True,
+ )
+
+ download_tag = soup.select_one(Selector.DROPDOWN_MENU) or css.select_one(soup, Selector.DOWNLOAD_BUTTON)
+
+ # Manually parse link. Some URLs are invalid. ex: https://cyberfile.me/7cfu
+ # For the download URL, the slug does not actually matter. It can be anything
+ raw_link = get_text_between(css.get_attr(download_tag, "onclick"), "('", "');")
+ token = raw_link.rpartition("?download_token=")[-1]
+ link = self.parse_url(raw_link).with_query(download_token=token)
+
+ scrape_item.possible_datetime = self.parse_date(
+ css.select_one_get_text(soup, Selector.FILE_UPLOAD_DATE), "%d/%m/%Y %H:%M:%S"
+ )
+
+ filename = css.select_one_get_text(soup, Selector.FILE_NAME)
+ custom_filename, ext = self.get_filename_and_ext(filename)
+ await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
+
+ _handle_content_id_task = auto_task_id(_handle_content_id)
+
+ async def _get_soup_from_ajax_api(
+ self, scrape_item: ScrapeItem, data: dict[str, Any], *, is_file: bool = False
+ ) -> BeautifulSoup:
+ async def ajax_api_request() -> BeautifulSoup:
+ ajax_url = self.FILE_API_URL if is_file else self.FOLDERS_API_URL
async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.DOMAIN, ajax_url, data=data)
- html: str = json_resp["html"]
- return BeautifulSoup(html.replace("\\", ""), "html.parser"), json_resp["page_title"]
-
- password = scrape_item.url.query.get("password", "")
- ajax_url = self.files_api_url if is_file else self.folders_api_url
- ajax_soup, ajax_title = await get_ajax_info()
- if is_password_protected(ajax_soup):
- if not password:
- raise PasswordProtectedError
- try:
- node_id = data.get("nodeId") or ajax_soup.select(_SELECTOR.FOLDER_ID)[0]["value"]
- except (IndexError, AttributeError):
- check_soup_error(ajax_soup)
- raise ScrapeError(422, message="nodeId not found") from None
-
- # Make a request with the password. Access to the file/folder will be stored in cookies
- pw_data = {"folderPassword": password, "folderId": node_id, "submitme": 1}
- async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.DOMAIN, self.folder_password_api_url, data=pw_data)
- if not json_resp.get("success"):
- raise PasswordProtectedError(message="Incorrect password")
-
- ajax_soup, ajax_title = await get_ajax_info()
-
- return ajax_soup, ajax_title
-
-
-def is_password_protected(soup: BeautifulSoup) -> bool:
- html = soup.get_text()
- return any(text in html for text in ("Enter File Password", "Password Required"))
-
-
-def check_soup_error(soup: BeautifulSoup) -> None:
- html = soup.get_text()
- for code, errors in SOUP_ERRORS.items():
- for text in errors:
- if text in html:
- raise ScrapeError(code)
+ json_resp: dict[str, str] = await self.client.post_data(
+ self.DOMAIN,
+ ajax_url,
+ data=data,
+ headers={"X-Requested-With": "XMLHttpRequest"},
+ )
+
+ return BeautifulSoup(json_resp["html"].replace("\\", ""), "html.parser")
+
+ soup = await ajax_api_request()
+ if soup.select_one(Selector.PASSWORD_PROTECTED):
+ node_id: str = data.get("nodeId") or css.select_one_get_attr(soup, Selector.FOLDER_ID, "value")
+ await self._unlock_password_protected_folder(scrape_item, node_id)
+ soup = await ajax_api_request()
+
+ _check_is_available(soup)
+ return soup
+
+ async def _unlock_password_protected_file(self, scrape_item: ScrapeItem, file_id: str) -> BeautifulSoup:
+ password = scrape_item.pop_query("password")
+ if not password:
+ raise PasswordProtectedError
+
+ password_post_url = (self.PRIMARY_URL / file_id).with_query("pt=")
+ async with self.request_limiter:
+ resp_bytes = await self.client.post_data_raw(
+ self.DOMAIN,
+ password_post_url,
+ data={"filePassword": password, "submitme": 1},
+ )
+ soup = BeautifulSoup(resp_bytes, "html.parser")
+
+ if soup.select_one(Selector.PASSWORD_PROTECTED):
+ raise PasswordProtectedError("File password is invalid")
+ return soup
+
+ async def _unlock_password_protected_folder(self, scrape_item: ScrapeItem, node_id: str) -> None:
+ password = scrape_item.pop_query("password")
+ if not password:
+ raise PasswordProtectedError
+
+ # Make a request with the password. Access to the file/folder will be stored in cookies
+ async with self.request_limiter:
+ json_resp: dict = await self.client.post_data(
+ self.DOMAIN,
+ self.FOLDER_PASSWORD_API_URL,
+ data={
+ "folderPassword": password,
+ "folderId": node_id,
+ "submitme": 1,
+ },
+ )
+ if not json_resp.get("success"):
+ raise PasswordProtectedError(message="Incorrect password")
+
+
+def _check_is_available(soup: BeautifulSoup):
+ if soup.select(Selector.RECAPTCHA):
+ raise DDOSGuardError("Google recaptcha found")
+
+ content = soup.get_text()
+
+ if "File has been removed" in content:
+ raise ScrapeError(410)
+
+ if "File is not publicly available" in content:
+ raise ScrapeError(401)
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index 4eab1539e..fd0c989d9 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -190,6 +190,7 @@ def get_filename_and_ext(filename: str, forum: bool = False) -> tuple[str, str]:
filename_as_path = filename_as_path.with_suffix(filename_as_path.suffix.lower())
filename_as_str = truncate_str(filename_as_path.stem.removesuffix(".")) + filename_as_path.suffix
filename_as_path = Path(sanitize_filename(filename_as_str))
+ filename_as_path = Path(filename_as_path.stem.strip() + filename_as_path.suffix)
return filename_as_path.name, filename_as_path.suffix
diff --git a/tests/crawlers/test_cases/cyberfile.py b/tests/crawlers/test_cases/cyberfile.py
new file mode 100644
index 000000000..11fc30752
--- /dev/null
+++ b/tests/crawlers/test_cases/cyberfile.py
@@ -0,0 +1,130 @@
+import re
+
+DOMAIN = "cyberfile"
+TEST_CASES = [
+ (
+ "https://cyberfile.me/c5w9",
+ [
+ {
+ "url": r"re:https\:\/\/p2\.cyberfile\.me/c5w9/ANGEL_MOVIE_082\.mp4\?download_token\=",
+ "filename": "ANGEL_MOVIE_082.mp4",
+ "referer": "https://cyberfile.me/c5w9",
+ "album_id": None,
+ "datetime": 1726579084,
+ }
+ ],
+ ),
+ (
+ "https://cyberfile.me/shared/-dprdwxcs8",
+ [
+ {
+ "url": r"re:https\:\/\/p2\.cyberfile\.me/88pe/WAAA-127\.mp4\?download_token\=",
+ "filename": "WAAA-127.mp4",
+ "referer": "https://cyberfile.me/88pe",
+ "download_folder": r"re:JAV \(Cyberfile\)",
+ "album_id": "-dprdwxcs8",
+ "datetime": 1728519855,
+ }
+ ],
+ ),
+ (
+ "https://cyberfile.me/b8lt",
+ [
+ {
+ "url": r"re:https\:\/\/p1\.cyberfile\.me/b8lt/Sophia_Locke__-_Inserted_\(4K\).mp4\?download_token\=",
+ "filename": "Sophia Locke - Inserted (4K).mp4",
+ "original_filename": "Sophia Locke - Inserted (4K).mp4",
+ "referer": "https://cyberfile.me/b8lt",
+ "album_id": None,
+ "datetime": 1696232232,
+ }
+ ],
+ ),
+ (
+ "https://cyberfile.me/shared/avk-q2uqnz",
+ [
+ {
+ "url": r"re:https\:\/\/p1\.cyberfile\.me/71nn/tabooheat\.23\.03\.29\.cory\.chase\.and\.sophia\.locke\.4k\.mp4\?download_token\=",
+ "filename": "tabooheat.23.03.29.cory.chase.and.sophia.locke.4k.mp4",
+ "referer": "https://cyberfile.me/71nn",
+ "download_folder": r"re:Sophia Locke \(Cyberfile\)",
+ "album_id": "avk-q2uqnz",
+ "datetime": 1698316590,
+ }
+ ],
+ 44,
+ ),
+ (
+ # Password protected folder, no password supplied
+ "https://cyberfile.me/folder/30cab170dd652309c8b8f25aaba4ec85/Autumn_Falls_JulesJordan_Videos",
+ [],
+ ),
+ (
+ # Password protected folder, invalid password supplied
+ "https://cyberfile.me/folder/30cab170dd652309c8b8f25aaba4ec85/Autumn_Falls_JulesJordan_Videos?password=1234",
+ [],
+ ),
+ (
+ # Password protected folder, valid password supplied
+ "https://cyberfile.me/folder/30cab170dd652309c8b8f25aaba4ec85/Autumn_Falls_JulesJordan_Videos?password=studio",
+ [
+ {
+ "url": r"re:https\:\/\/p1\.cyberfile\.me/8QdR/www\.0xxx\.ws_JulesJordan\.20\.03\.03\.Autumn\.Falls\.XXX\.1080p\.MP4-KTR\.mp4\?download_token\=",
+ "filename": "www.0xxx.ws_JulesJordan.20.03.03.Autumn.Falls.XXX.1080p.MP4-KTR.mp4",
+ "referer": "https://cyberfile.me/8QdR",
+ "download_folder": r"re:Autumn Falls JulesJordan Videos \(Cyberfile\)",
+ "album_id": "30cab170dd652309c8b8f25aaba4ec85",
+ "datetime": 1669199683,
+ }
+ ],
+ 6,
+ ),
+ (
+ # Password protected file
+ "https://cyberfile.me/ktP2",
+ [],
+ ),
+ (
+ # Password protected file
+ "https://cyberfile.me/ktP2?password=pass",
+ [
+ {
+ "url": r"re:https\:\/\/p2\.cyberfile\.me/ktP2/HIPANGEL_021\.mp4\?download_token\=",
+ "filename": "HIPANGEL_021.mp4",
+ "referer": "https://cyberfile.me/ktP2",
+ "album_id": None,
+ "datetime": 1726582638,
+ }
+ ],
+ ),
+ ( # File with malformed download URL ( in the slug)
+ "https://cyberfile.me/7cfu",
+ [
+ {
+ "url": r"re:https\:\/\/p1\.cyberfile\.me/7cfu/1_Hour_ASMR___\?download_token\=",
+ "filename": "1 Hour ASMR.mp4",
+ "original_filename": "1 Hour ASMR ???? ????.mp4",
+ "referer": "https://cyberfile.me/7cfu",
+ "album_id": None,
+ "datetime": 1703854148,
+ }
+ ],
+ ),
+ ( # File with emojis in the name
+ # This will fail on Windows and MacOS
+ "https://cyberfile.me/7cfq",
+ [
+ {
+ "url": "re:"
+ + re.escape(
+ "https://p1.cyberfile.me/7cfq/ASMR___1_HOUR_OF_UP-CLOSE_&_PERSONAL_ATTENTION_%E2%9D%A4%EF%B8%8F_NEGATIVE_ENERGY_&_STRESS_REMOVAL_%E2%9D%A4%EF%B8%8F_(LIGHT_LOFI).mp4?download_token="
+ ),
+ "filename": "ASMR 1 HOUR OF UP-CLOSE & PERSONAL ATTENTION ❤️ NEGATIVE ENERGY & STRESS REMOVAL ❤️ (LIGHT LO.mp4",
+ "original_filename": "ASMR 1 HOUR OF UP-CLOSE & PERSONAL ATTENTION ❤️ NEGATIVE ENERGY & STRESS REMOVAL ❤️ (LIGHT LOFI).mp4",
+ "referer": "https://cyberfile.me/7cfq",
+ "album_id": None,
+ "datetime": 1703853101,
+ }
+ ],
+ ),
+]
diff --git a/tests/crawlers/test_cases/iceyfile.py b/tests/crawlers/test_cases/iceyfile.py
new file mode 100644
index 000000000..59722fb49
--- /dev/null
+++ b/tests/crawlers/test_cases/iceyfile.py
@@ -0,0 +1,31 @@
+import re
+
+DOMAIN = "iceyfile"
+TEST_CASES = [
+ # This will fail. iceyfile now always requires a 1 type access only captcha ofr individual files
+ (
+ "https://iceyfile.com/be60cca8a9dec177/fff.png",
+ [
+ {
+ "url": "re:" + re.escape("https://srv1.iceyfile.net/be60cca8a9dec177/fff.png?download_token="),
+ "filename": "fff",
+ "referer": "https://iceyfile.com/be60cca8a9dec177/fff.png",
+ "album_id": None,
+ "datetime": 1755488372,
+ }
+ ],
+ ),
+ (
+ "https://iceyfile.com/folder/75061972f799eeacba32ac81f37493bc/CDL_test",
+ [
+ {
+ "url": "re:" + re.escape("https://srv1.iceyfile.net/be60cca8a9dec177/fff.png?download_token="),
+ "filename": "fff.png",
+ "referer": "https://iceyfile.com/be60cca8a9dec177/fff.png",
+ "download_folder": r"re:CDL_test \(Iceyfile\)",
+ "album_id": "75061972f799eeacba32ac81f37493bc",
+ "datetime": 1755491602,
+ }
+ ],
+ ),
+]