Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: nested shared folders (yetishare) (#1242)
NTFSvolumebbf6bf7e0ba7
diff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.pyindex 666786184..18f5dd46b 100644--- a/cyberdrop_dl/crawlers/_yetishare.py+++ b/cyberdrop_dl/crawlers/_yetishare.py@@ -1,13 +1,13 @@from __future__ import annotationsimport itertools-from typing import TYPE_CHECKING, ClassVar+from typing import TYPE_CHECKING, Any, ClassVarfrom aiolimiter import AsyncLimiterfrom bs4 import BeautifulSoup-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths-from cyberdrop_dl.exceptions import PasswordProtectedError, ScrapeError+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id+from cyberdrop_dl.exceptions import DDOSGuardError, PasswordProtectedError, ScrapeErrorfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between@@ -15,237 +15,237 @@ if TYPE_CHECKING:from cyberdrop_dl.data_structures.url_objects import ScrapeItem-SOUP_ERRORS = {- 410: ["File has been removed"],- 401: ["File is not publicly available"],-}---class Selectors:- DOWNLOAD_BUTTON = 'div[class="btn-group responsiveMobileMargin"] button'- FILE_MENU = 'ul[class="dropdown-menu dropdown-info account-dropdown-resize-menu"] li a'- FILE_NAME = "div.image-name-title"+class Selector:+ DOWNLOAD_BUTTON = ".btn-group.responsiveMobileMargin button:contains('Download')[onclick*='download_token']"+ DROPDOWN_MENU = ".dropdown-menu.dropdown-info a[onclick*='download_token']"+ FILE_NAME = ".image-name-title"FILE_UPLOAD_DATE = "td:contains('Uploaded:') + td"- FOLDER_ID_JS = "div[class*='page-container'] script:contains('loadImages')"- FOLDER_ID = "#folderId"- FOLDER_ITEM = "div[class=fileListing] div[class*=fileItem]"- FOLDER_N_PAGES = "a[onclick*=loadImages]"- LOGIN_FORM = "form[id=form_login]"- PASSWORD_FORM = "form[method='POST']"- SHARED_N_PAGES = "input[id=rspTotalPages]"- SHOW_FILE_INFO_JS = "script:contains('showFileInformation')"+ FILE_INFO = "script:contains('showFileInformation')"+ LOAD_IMAGES = "div[class*='page-container'] script:contains('loadImages')"+ FOLDER_ID = "#folderId"+ _FOLDER_ITEM = "#fileListing [class*=fileItem]"+ FILES = f"{_FOLDER_ITEM}[fileid]"+ SUBFOLDERS = f"{_FOLDER_ITEM}[folderid]"+ FOLDER_TOTAL_PAGES = "input#rspTotalPages"-_SELECTOR = Selectors()+ LOGIN_FORM = "form#form_login"+ PASSWORD_PROTECTED = "#folderPasswordForm, #filePassword"+ RECAPTCHA = "form[method=POST] script[src*='/recaptcha/api.js']"class YetiShareCrawler(Crawler, is_abc=True):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Files": "/...",- "Folder": "/folder/...",- "Shared folder": "/shared/...",+ "Files": (+ "/<file_id>",+ "/<file_id>/<file_name>",+ ),+ "Public Folders": (+ "/folder/<folder_id>",+ "/folder/<folder_id>/<folder_name>",+ ),+ "Shared folders": "/shared/<share_key>",}def __init_subclass__(cls, **kwargs) -> None:super().__init_subclass__(**kwargs)- cls.folders_api_url = cls.PRIMARY_URL / "account/ajax/load_files"- cls.files_api_url = cls.PRIMARY_URL / "account/ajax/file_details"- cls.folder_password_api_url = cls.PRIMARY_URL / "ajax/folder_password_process"+ cls.FOLDERS_API_URL = cls.PRIMARY_URL / "account/ajax/load_files"+ cls.FILE_API_URL = cls.PRIMARY_URL / "account/ajax/file_details"+ cls.FOLDER_PASSWORD_API_URL = cls.PRIMARY_URL / "ajax/folder_password_process"def __post_init__(self) -> None:self.request_limiter = AsyncLimiter(5, 1)async def fetch(self, scrape_item: ScrapeItem) -> None:- if "folder" in scrape_item.url.parts:- return await self.folder(scrape_item)- if "shared" in scrape_item.url.parts:- return await self.shared_folder(scrape_item)- return await self.file(scrape_item)+ match scrape_item.url.parts[1:]:+ case ["folder", folder_id, *_]:+ return await self.folder(scrape_item, folder_id)+ case ["shared", folder_id]:+ return await self.folder(scrape_item, folder_id, is_shared=True)+ case [file_id]:+ return await self.file(scrape_item, file_id)+ case [file_id, _]:+ return await self.file(scrape_item, file_id)+ case _:+ raise ValueError@error_handling_wrapper- async def folder(self, scrape_item: ScrapeItem) -> None:+ async def folder(self, scrape_item: ScrapeItem, folder_id: str, is_shared: bool = False) -> None:+ # Make request to update cookies. Access to folders in saved on cookiesasync with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- if soup.select_one(_SELECTOR.LOGIN_FORM):+ if soup.select_one(Selector.LOGIN_FORM):raise ScrapeError(410, "Folder has been deleted")- js_text = soup.select(_SELECTOR.FOLDER_ID_JS)[-1].text- # ex: loadImages('folder', '12345', 1, 0, '', {'searchTerm': "", 'filterUploadedDateRange': ""});- js_text = get_text_between(js_text, "loadImages(", ");")- _, node_id = js_text.replace("'", "").split(",", 1)- album_id = scrape_item.url.parts[2]- title: str = ""- default_data = {"pageType": "folder", "perPage": 0, "filterOrderBy": ""}- n_pages = 1- for page in itertools.count(1):- data = default_data | {"nodeId": node_id, "pageStart": page}- ajax_soup, ajax_title = await self.get_soup_from_ajax(data, scrape_item)- if not title:- title = self.create_title(ajax_title, album_id)- scrape_item.setup_as_album(title, album_id=album_id)- next_page_tag = ajax_soup.select(_SELECTOR.FOLDER_N_PAGES)[-1]- n_pages_text: str = css.get_attr(next_page_tag, "onclick")- n_pages = int(n_pages_text.split(",")[2].split(")")[0].strip())-- _ = self.iter_files(scrape_item, ajax_soup)- if page >= n_pages:- break+ if is_shared:+ page_type = "nonaccountshared"+ node_id = ""- @error_handling_wrapper- async def shared_folder(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- await self.client.get_soup(self.DOMAIN, scrape_item.url)+ else:+ # ex: loadImages('folder', '12345', 1, 0, '', {'searchTerm': "", 'filterUploadedDateRange': ""});+ page_type = "folder"+ load_images = get_text_between(soup.select(Selector.LOAD_IMAGES)[-1].text, "loadImages(", ");")+ node_id = load_images.replace("'", "").split(",")[1].strip()- subfolders = []- node_id = ""- album_id = scrape_item.url.parts[2]page = 1- default_data = {"pageType": "nonaccountshared", "perPage": 0, "filterOrderBy": ""}- n_pages = 1- while True:- data = default_data | {"nodeId": node_id, "pageStart": page}- ajax_soup, ajax_title = await self.get_soup_from_ajax(data, scrape_item)- if page == 1:- title = self.create_title(ajax_title, album_id)- scrape_item.setup_as_album(title, album_id=album_id)- n_pages = int(css.select_one_get_attr(ajax_soup, _SELECTOR.SHARED_N_PAGES, "value"))-- subfolders.extend(self.iter_files(scrape_item, ajax_soup, iter_subfolders=False))- page += 1- if page > n_pages:- if not subfolders:- break- node_id = str(subfolders.pop(0))- page = 1+ total_pages = None+ if not scrape_item.album_id:+ scrape_item.setup_as_album("", album_id=folder_id)++ if not is_shared:+ title = css.page_title(soup, self.DOMAIN).removesuffix("Folder").strip()+ scrape_item.add_to_parent_title(self.create_title(title, folder_id))++ for page in itertools.count(1):+ ajax_soup = await self._get_soup_from_ajax_api(+ scrape_item,+ data={+ "pageType": page_type,+ "perPage": 0,+ "filterOrderBy": "",+ "nodeId": node_id,+ "pageStart": page,+ },+ )++ if total_pages is None:+ total_pages = int(css.select_one_get_attr(ajax_soup, Selector.FOLDER_TOTAL_PAGES, "value"))++ for file in ajax_soup.select(Selector.FILES):+ file_url = self.parse_url(css.get_attr(file, "dtfullurl"))+ content_id = css.get_attr(file, "fileid")+ new_scrape_item = scrape_item.create_child(file_url)+ self.create_task(self._handle_content_id_task(new_scrape_item, content_id))+ scrape_item.add_children()++ for _, new_scrape_item in self.iter_children(+ scrape_item, ajax_soup, Selector.SUBFOLDERS, attribute="sharing-url"+ ):+ self.create_task(self.run(new_scrape_item))++ if page >= total_pages:+ break@error_handling_wrapper- async def file(self, scrape_item: ScrapeItem) -> None:- def get_content_id(soup: BeautifulSoup) -> int | None:- if file_info := soup.select_one(_SELECTOR.SHOW_FILE_INFO_JS):- content_id = get_text_between(file_info.text, "showFileInformation(", ");")- return int(content_id)-- file_id = scrape_item.url.parts[1]- canonical_url = self.PRIMARY_URL / file_id- if await self.check_complete_from_referer(canonical_url):+ async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:+ if await self.check_complete_from_referer(scrape_item):return- password = scrape_item.url.query.get("password", "")- scrape_item.url = canonical_urlasync with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)- if is_password_protected(soup):- form = soup.select_one(_SELECTOR.PASSWORD_FORM)- if not form:- raise PasswordProtectedError("Unable to parse Password Protected File details")+ if soup.select_one(Selector.PASSWORD_PROTECTED):+ soup = await self._unlock_password_protected_file(scrape_item, file_id)- password_post_url = self.parse_url(css.get_attr(form, "action"))- data = {"filePassword": password, "submitme": 1}- async with self.request_limiter:- resp_bytes = await self.client.post_data_raw(self.DOMAIN, password_post_url, data=data)-- soup = BeautifulSoup(resp_bytes, "html.parser")- if is_password_protected(soup):- raise PasswordProtectedError("File password is invalid")+ _check_is_available(soup)- if content_id := get_content_id(soup):- return await self.handle_content_id(scrape_item, content_id)+ content_id = int(+ get_text_between(+ css.select_one_get_text(soup, Selector.FILE_INFO),+ "showFileInformation(",+ ");",+ )+ )- check_soup_error(soup)- raise ScrapeError(422, message="contentId not found")+ return await self._handle_content_id(scrape_item, content_id)@error_handling_wrapper- async def handle_content_id(self, scrape_item: ScrapeItem, content_id: int) -> None:- data = {"u": content_id}- ajax_soup, page_title = await self.get_soup_from_ajax(data, scrape_item, is_file=True)-- try:- file_tag = ajax_soup.select_one(_SELECTOR.FILE_MENU) or ajax_soup.select(_SELECTOR.DOWNLOAD_BUTTON)[-1]- html_download_text = css.get_attr(file_tag, "onclick")- link_str = html_download_text.split("'")[1].strip().removesuffix("'")- link = self.parse_url(link_str)- except (AttributeError, IndexError, KeyError):- check_soup_error(ajax_soup)- raise ScrapeError(422, "Couldn't find download button") from None-- if uploaded_date := ajax_soup.select_one(_SELECTOR.FILE_UPLOAD_DATE):- scrape_item.possible_datetime = self.parse_date(uploaded_date.text.strip(), "%d/%m/%Y %H:%M:%S")-- if ajax_title := ajax_soup.select_one(_SELECTOR.FILE_NAME):- filename = ajax_title.get_text(strip=True)- else:- filename = page_title.strip()-- filename, ext = self.get_filename_and_ext(filename or link.name)- await self.handle_file(link, scrape_item, filename, ext)-- def iter_files(self, scrape_item: ScrapeItem, soup: BeautifulSoup, *, iter_subfolders: bool = True) -> list[str]:- """Proccess all the files in this folder. Optionally process subfolders-- Returns a list with the `folder_id` of every subfolder"""- folder_ids = []- for item in soup.select(_SELECTOR.FOLDER_ITEM):- if folder_id := css.get_attr_or_none(item, "folderid"):- folder_ids.append(folder_id)- if not iter_subfolders:- continue- link_str = css.get_attr(item, "sharing-url")- elif css.get_attr_or_none(item, "fileid"):- link_str = css.get_attr(item, "dtfullurl")- else:- continue-- link = self.parse_url(link_str)- new_scrape_item = scrape_item.create_child(link)- self.manager.task_group.create_task(self.run(new_scrape_item))- scrape_item.add_children()- return folder_ids-- async def get_soup_from_ajax(- self, data: dict, scrape_item: ScrapeItem, is_file: bool = False- ) -> tuple[BeautifulSoup, str]:- """Returns soup and page title as a tuple"""-- async def get_ajax_info() -> tuple[BeautifulSoup, str]:+ async def _handle_content_id(self, scrape_item: ScrapeItem, content_id: int) -> None:+ soup = await self._get_soup_from_ajax_api(+ scrape_item,+ data={"u": content_id},+ is_file=True,+ )++ download_tag = soup.select_one(Selector.DROPDOWN_MENU) or css.select_one(soup, Selector.DOWNLOAD_BUTTON)++ # Manually parse link. Some URLs are invalid. ex: https://cyberfile.me/7cfu+ # For the download URL, the slug does not actually matter. It can be anything+ raw_link = get_text_between(css.get_attr(download_tag, "onclick"), "('", "');")+ token = raw_link.rpartition("?download_token=")[-1]+ link = self.parse_url(raw_link).with_query(download_token=token)++ scrape_item.possible_datetime = self.parse_date(+ css.select_one_get_text(soup, Selector.FILE_UPLOAD_DATE), "%d/%m/%Y %H:%M:%S"+ )++ filename = css.select_one_get_text(soup, Selector.FILE_NAME)+ custom_filename, ext = self.get_filename_and_ext(filename)+ await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)++ _handle_content_id_task = auto_task_id(_handle_content_id)++ async def _get_soup_from_ajax_api(+ self, scrape_item: ScrapeItem, data: dict[str, Any], *, is_file: bool = False+ ) -> BeautifulSoup:+ async def ajax_api_request() -> BeautifulSoup:+ ajax_url = self.FILE_API_URL if is_file else self.FOLDERS_API_URLasync with self.request_limiter:- json_resp: dict = await self.client.post_data(self.DOMAIN, ajax_url, data=data)- html: str = json_resp["html"]- return BeautifulSoup(html.replace("\\", ""), "html.parser"), json_resp["page_title"]-- password = scrape_item.url.query.get("password", "")- ajax_url = self.files_api_url if is_file else self.folders_api_url- ajax_soup, ajax_title = await get_ajax_info()- if is_password_protected(ajax_soup):- if not password:- raise PasswordProtectedError- try:- node_id = data.get("nodeId") or ajax_soup.select(_SELECTOR.FOLDER_ID)[0]["value"]- except (IndexError, AttributeError):- check_soup_error(ajax_soup)- raise ScrapeError(422, message="nodeId not found") from None-- # Make a request with the password. Access to the file/folder will be stored in cookies- pw_data = {"folderPassword": password, "folderId": node_id, "submitme": 1}- async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.DOMAIN, self.folder_password_api_url, data=pw_data)- if not json_resp.get("success"):- raise PasswordProtectedError(message="Incorrect password")-- ajax_soup, ajax_title = await get_ajax_info()-- return ajax_soup, ajax_title---def is_password_protected(soup: BeautifulSoup) -> bool:- html = soup.get_text()- return any(text in html for text in ("Enter File Password", "Password Required"))---def check_soup_error(soup: BeautifulSoup) -> None:- html = soup.get_text()- for code, errors in SOUP_ERRORS.items():- for text in errors:- if text in html:- raise ScrapeError(code)+ json_resp: dict[str, str] = await self.client.post_data(+ self.DOMAIN,+ ajax_url,+ data=data,+ headers={"X-Requested-With": "XMLHttpRequest"},+ )++ return BeautifulSoup(json_resp["html"].replace("\\", ""), "html.parser")++ soup = await ajax_api_request()+ if soup.select_one(Selector.PASSWORD_PROTECTED):+ node_id: str = data.get("nodeId") or css.select_one_get_attr(soup, Selector.FOLDER_ID, "value")+ await self._unlock_password_protected_folder(scrape_item, node_id)+ soup = await ajax_api_request()++ _check_is_available(soup)+ return soup++ async def _unlock_password_protected_file(self, scrape_item: ScrapeItem, file_id: str) -> BeautifulSoup:+ password = scrape_item.pop_query("password")+ if not password:+ raise PasswordProtectedError++ password_post_url = (self.PRIMARY_URL / file_id).with_query("pt=")+ async with self.request_limiter:+ resp_bytes = await self.client.post_data_raw(+ self.DOMAIN,+ password_post_url,+ data={"filePassword": password, "submitme": 1},+ )+ soup = BeautifulSoup(resp_bytes, "html.parser")++ if soup.select_one(Selector.PASSWORD_PROTECTED):+ raise PasswordProtectedError("File password is invalid")+ return soup++ async def _unlock_password_protected_folder(self, scrape_item: ScrapeItem, node_id: str) -> None:+ password = scrape_item.pop_query("password")+ if not password:+ raise PasswordProtectedError++ # Make a request with the password. Access to the file/folder will be stored in cookies+ async with self.request_limiter:+ json_resp: dict = await self.client.post_data(+ self.DOMAIN,+ self.FOLDER_PASSWORD_API_URL,+ data={+ "folderPassword": password,+ "folderId": node_id,+ "submitme": 1,+ },+ )+ if not json_resp.get("success"):+ raise PasswordProtectedError(message="Incorrect password")+++def _check_is_available(soup: BeautifulSoup):+ if soup.select(Selector.RECAPTCHA):+ raise DDOSGuardError("Google recaptcha found")++ content = soup.get_text()++ if "File has been removed" in content:+ raise ScrapeError(410)++ if "File is not publicly available" in content:+ raise ScrapeError(401)diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex 4eab1539e..fd0c989d9 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -190,6 +190,7 @@ def get_filename_and_ext(filename: str, forum: bool = False) -> tuple[str, str]:filename_as_path = filename_as_path.with_suffix(filename_as_path.suffix.lower())filename_as_str = truncate_str(filename_as_path.stem.removesuffix(".")) + filename_as_path.suffixfilename_as_path = Path(sanitize_filename(filename_as_str))+ filename_as_path = Path(filename_as_path.stem.strip() + filename_as_path.suffix)return filename_as_path.name, filename_as_path.suffixdiff --git a/tests/crawlers/test_cases/cyberfile.py b/tests/crawlers/test_cases/cyberfile.pynew file mode 100644index 000000000..11fc30752--- /dev/null+++ b/tests/crawlers/test_cases/cyberfile.py@@ -0,0 +1,130 @@+import re++DOMAIN = "cyberfile"+TEST_CASES = [+ (+ "https://cyberfile.me/c5w9",+ [+ {+ "url": r"re:https\:\/\/p2\.cyberfile\.me/c5w9/ANGEL_MOVIE_082\.mp4\?download_token\=",+ "filename": "ANGEL_MOVIE_082.mp4",+ "referer": "https://cyberfile.me/c5w9",+ "album_id": None,+ "datetime": 1726579084,+ }+ ],+ ),+ (+ "https://cyberfile.me/shared/-dprdwxcs8",+ [+ {+ "url": r"re:https\:\/\/p2\.cyberfile\.me/88pe/WAAA-127\.mp4\?download_token\=",+ "filename": "WAAA-127.mp4",+ "referer": "https://cyberfile.me/88pe",+ "download_folder": r"re:JAV \(Cyberfile\)",+ "album_id": "-dprdwxcs8",+ "datetime": 1728519855,+ }+ ],+ ),+ (+ "https://cyberfile.me/b8lt",+ [+ {+ "url": r"re:https\:\/\/p1\.cyberfile\.me/b8lt/Sophia_Locke__-_Inserted_\(4K\).mp4\?download_token\=",+ "filename": "Sophia Locke - Inserted (4K).mp4",+ "original_filename": "Sophia Locke - Inserted (4K).mp4",+ "referer": "https://cyberfile.me/b8lt",+ "album_id": None,+ "datetime": 1696232232,+ }+ ],+ ),+ (+ "https://cyberfile.me/shared/avk-q2uqnz",+ [+ {+ "url": r"re:https\:\/\/p1\.cyberfile\.me/71nn/tabooheat\.23\.03\.29\.cory\.chase\.and\.sophia\.locke\.4k\.mp4\?download_token\=",+ "filename": "tabooheat.23.03.29.cory.chase.and.sophia.locke.4k.mp4",+ "referer": "https://cyberfile.me/71nn",+ "download_folder": r"re:Sophia Locke \(Cyberfile\)",+ "album_id": "avk-q2uqnz",+ "datetime": 1698316590,+ }+ ],+ 44,+ ),+ (+ # Password protected folder, no password supplied+ "https://cyberfile.me/folder/30cab170dd652309c8b8f25aaba4ec85/Autumn_Falls_JulesJordan_Videos",+ [],+ ),+ (+ # Password protected folder, invalid password supplied+ "https://cyberfile.me/folder/30cab170dd652309c8b8f25aaba4ec85/Autumn_Falls_JulesJordan_Videos?password=1234",+ [],+ ),+ (+ # Password protected folder, valid password supplied+ "https://cyberfile.me/folder/30cab170dd652309c8b8f25aaba4ec85/Autumn_Falls_JulesJordan_Videos?password=studio",+ [+ {+ "url": r"re:https\:\/\/p1\.cyberfile\.me/8QdR/www\.0xxx\.ws_JulesJordan\.20\.03\.03\.Autumn\.Falls\.XXX\.1080p\.MP4-KTR\.mp4\?download_token\=",+ "filename": "www.0xxx.ws_JulesJordan.20.03.03.Autumn.Falls.XXX.1080p.MP4-KTR.mp4",+ "referer": "https://cyberfile.me/8QdR",+ "download_folder": r"re:Autumn Falls JulesJordan Videos \(Cyberfile\)",+ "album_id": "30cab170dd652309c8b8f25aaba4ec85",+ "datetime": 1669199683,+ }+ ],+ 6,+ ),+ (+ # Password protected file+ "https://cyberfile.me/ktP2",+ [],+ ),+ (+ # Password protected file+ "https://cyberfile.me/ktP2?password=pass",+ [+ {+ "url": r"re:https\:\/\/p2\.cyberfile\.me/ktP2/HIPANGEL_021\.mp4\?download_token\=",+ "filename": "HIPANGEL_021.mp4",+ "referer": "https://cyberfile.me/ktP2",+ "album_id": None,+ "datetime": 1726582638,+ }+ ],+ ),+ ( # File with malformed download URL ( in the slug)+ "https://cyberfile.me/7cfu",+ [+ {+ "url": r"re:https\:\/\/p1\.cyberfile\.me/7cfu/1_Hour_ASMR___\?download_token\=",+ "filename": "1 Hour ASMR.mp4",+ "original_filename": "1 Hour ASMR ???? ????.mp4",+ "referer": "https://cyberfile.me/7cfu",+ "album_id": None,+ "datetime": 1703854148,+ }+ ],+ ),+ ( # File with emojis in the name+ # This will fail on Windows and MacOS+ "https://cyberfile.me/7cfq",+ [+ {+ "url": "re:"+ + re.escape(+ "https://p1.cyberfile.me/7cfq/ASMR___1_HOUR_OF_UP-CLOSE_&_PERSONAL_ATTENTION_%E2%9D%A4%EF%B8%8F_NEGATIVE_ENERGY_&_STRESS_REMOVAL_%E2%9D%A4%EF%B8%8F_(LIGHT_LOFI).mp4?download_token="+ ),+ "filename": "ASMR 1 HOUR OF UP-CLOSE & PERSONAL ATTENTION ❤️ NEGATIVE ENERGY & STRESS REMOVAL ❤️ (LIGHT LO.mp4",+ "original_filename": "ASMR 1 HOUR OF UP-CLOSE & PERSONAL ATTENTION ❤️ NEGATIVE ENERGY & STRESS REMOVAL ❤️ (LIGHT LOFI).mp4",+ "referer": "https://cyberfile.me/7cfq",+ "album_id": None,+ "datetime": 1703853101,+ }+ ],+ ),+]diff --git a/tests/crawlers/test_cases/iceyfile.py b/tests/crawlers/test_cases/iceyfile.pynew file mode 100644index 000000000..59722fb49--- /dev/null+++ b/tests/crawlers/test_cases/iceyfile.py@@ -0,0 +1,31 @@+import re++DOMAIN = "iceyfile"+TEST_CASES = [+ # This will fail. iceyfile now always requires a 1 type access only captcha ofr individual files+ (+ "https://iceyfile.com/be60cca8a9dec177/fff.png",+ [+ {+ "url": "re:" + re.escape("https://srv1.iceyfile.net/be60cca8a9dec177/fff.png?download_token="),+ "filename": "fff",+ "referer": "https://iceyfile.com/be60cca8a9dec177/fff.png",+ "album_id": None,+ "datetime": 1755488372,+ }+ ],+ ),+ (+ "https://iceyfile.com/folder/75061972f799eeacba32ac81f37493bc/CDL_test",+ [+ {+ "url": "re:" + re.escape("https://srv1.iceyfile.net/be60cca8a9dec177/fff.png?download_token="),+ "filename": "fff.png",+ "referer": "https://iceyfile.com/be60cca8a9dec177/fff.png",+ "download_folder": r"re:CDL_test \(Iceyfile\)",+ "album_id": "75061972f799eeacba32ac81f37493bc",+ "datetime": 1755491602,+ }+ ],+ ),+]