Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add safe.soul support (#1310)
NTFSvolume417aae7a80dc
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex e848a2a41..6d88707f1 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -87,6 +87,7 @@ from .rule34vault import Rule34VaultCrawlerfrom .rule34video import Rule34VideoCrawlerfrom .rule34xxx import Rule34XXXCrawlerfrom .rule34xyz import Rule34XYZCrawler+from .safe_soul import SafeSoulCrawlerfrom .saint import SaintCrawlerfrom .scrolller import ScrolllerCrawlerfrom .send_now import SendNowCrawlerdiff --git a/cyberdrop_dl/crawlers/_chibisafe.py b/cyberdrop_dl/crawlers/_chibisafe.pynew file mode 100644index 000000000..134ea0600--- /dev/null+++ b/cyberdrop_dl/crawlers/_chibisafe.py@@ -0,0 +1,100 @@+"""ChibiSafe (known as LoliSafe until v4.0.0)++https://github.com/chibisafe/chibisafe+https://chibisafe.moe/docs+https://chibisafe.app/++This is the file host framework used by bunkr, saint, cyberdrop, etc+"""++from __future__ import annotations++import dataclasses+import datetime # noqa: TC003+from typing import TYPE_CHECKING, Any, ClassVar++from pydantic import Field++from cyberdrop_dl.crawlers.crawler import Crawler+from cyberdrop_dl.models import AliasModel+from cyberdrop_dl.utils.dates import to_timestamp+from cyberdrop_dl.utils.utilities import error_handling_wrapper, type_adapter++if TYPE_CHECKING:+ from cyberdrop_dl.crawlers.crawler import SupportedPaths+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem++[email protected](slots=True)+class File:+ name: str+ url: str+ createdAt: datetime.datetime | None = None # noqa: N815+ original: str | None = None+++class Album(AliasModel):+ id: str = ""+ name: str = Field(validation_alias="title")+ files: list[File]+++_parse_file = type_adapter(File)+++class ChibiSafeCrawler(Crawler, is_abc=True):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Album": "/a/<album_id>",+ "File": "/<file_id>",+ }++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["a", album_id]:+ return await self.album(scrape_item, album_id)+ case [file_id]:+ return await self.file(scrape_item, file_id)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:+ content: dict[str, Any] = await self.request_json(self.PRIMARY_URL / "api/file" / file_id)+ file = _parse_file(content)+ self._handle_file(scrape_item, file)++ @error_handling_wrapper+ async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:+ content = await self.request_text(self.PRIMARY_URL / "api/album" / album_id)+ album = Album.model_validate_json(content, by_alias=True)+ album.id = album_id+ return await self._handle_album(scrape_item, album)++ async def _handle_album(self, scrape_item: ScrapeItem, album: Album) -> None:+ title = self.create_title(album.name, album.id)+ scrape_item.setup_as_album(title, album_id=album.id)+ results = await self.get_album_results(album.id)++ for file in album.files:+ url = self.parse_url(file.url.removeprefix("null"))+ if self.check_album_results(url, results):+ continue+ new_scrape_item = scrape_item.create_child(url)+ self._handle_file(new_scrape_item, file)+ scrape_item.add_children()++ @error_handling_wrapper+ def _handle_file(self, scrape_item: ScrapeItem, file: File) -> None:+ if scrape_item.possible_datetime is None and file.createdAt:+ scrape_item.possible_datetime = to_timestamp(file.createdAt)+ name = file.original or file.name+ filename, ext = self.get_filename_and_ext(name)+ self.create_task(+ self.handle_file(+ scrape_item.url,+ scrape_item,+ name,+ ext,+ custom_filename=filename,+ )+ )diff --git a/cyberdrop_dl/crawlers/safe_soul.py b/cyberdrop_dl/crawlers/safe_soul.pynew file mode 100644index 000000000..5384a9a65--- /dev/null+++ b/cyberdrop_dl/crawlers/safe_soul.py@@ -0,0 +1,55 @@+from __future__ import annotations++import datetime+from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl.crawlers._chibisafe import Album, ChibiSafeCrawler, File+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import css+from cyberdrop_dl.utils.utilities import error_handling_wrapper++if TYPE_CHECKING:+ import bs4++ from cyberdrop_dl.data_structures.url_objects import ScrapeItem+++class Selector:+ ALBUM_TITLE = "#title"+ FILE = "#table .image-container"+ FILE_DATE = ".details .file-date"+ FILE_NAME = ".details .name"+ FILE_URL = "a.image"+++class SafeSoulCrawler(ChibiSafeCrawler):+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://safe.soul.lol")+ DOMAIN: ClassVar[str] = "safe.soul.lol"+ FOLDER_DOMAIN: ClassVar[str] = "Safe.Soul"++ async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:+ # file endpoint is disabled+ return await self.direct_file(scrape_item)++ @error_handling_wrapper+ async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:+ # The album endpoint is enabled, but it returns incomplete info (ex: missing date)+ # So we scrape the HTML++ soup = await self.request_soup(scrape_item.url)+ album = Album(+ id=album_id,+ name=css.select_one_get_text(soup, Selector.ALBUM_TITLE),+ files=[_parse_file(tag) for tag in soup.select(Selector.FILE)],+ )+ return await self._handle_album(scrape_item, album)+++def _parse_file(file_tag: bs4.Tag) -> File:+ timestamp = int(css.select_one_get_attr(file_tag, Selector.FILE_DATE, "data-value"))++ return File(+ name=css.select_one_get_text(file_tag, Selector.FILE_NAME),+ url=css.select_one_get_attr(file_tag, Selector.FILE_URL, "href"),+ createdAt=datetime.datetime.fromtimestamp(timestamp),+ )diff --git a/tests/crawlers/test_cases/safe_soul.py b/tests/crawlers/test_cases/safe_soul.pynew file mode 100644index 000000000..22a35756b--- /dev/null+++ b/tests/crawlers/test_cases/safe_soul.py@@ -0,0 +1,44 @@+DOMAIN = "safe.soul.lol"+TEST_CASES = [+ (+ "https://safe.soul.lol/GGCW62sr.dat",+ [+ {+ "url": "https://safe.soul.lol/GGCW62sr.dat",+ "filename": "GGCW62sr.dat",+ "referer": "https://safe.soul.lol/GGCW62sr.dat",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ "https://safe.soul.lol/a/fH8aoyxD",+ [+ {+ "url": "https://safe.soul.lol/GGCW62sr.dat",+ "filename": "GGCW62sr.dat",+ "referer": "https://safe.soul.lol/GGCW62sr.dat",+ "album_id": "fH8aoyxD",+ "datetime": 1757997917,+ "download_folder": r"re:cdl_test \(Safe\.Soul\)",+ },+ {+ "url": "https://safe.soul.lol/rQPLKGeA.zip",+ "filename": "rQPLKGeA.zip",+ "referer": "https://safe.soul.lol/rQPLKGeA.zip",+ "album_id": "fH8aoyxD",+ "datetime": 1757997915,+ "download_folder": r"re:cdl_test \(Safe\.Soul\)",+ },+ {+ "url": "https://safe.soul.lol/8WXnedx1.bin",+ "filename": "8WXnedx1.bin",+ "referer": "https://safe.soul.lol/8WXnedx1.bin",+ "album_id": "fH8aoyxD",+ "datetime": 1757997914,+ "download_folder": r"re:cdl_test \(Safe\.Soul\)",+ },+ ],+ ),+]