Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add safe.soul support (#1310)

NTFSvolume417aae7a80dc
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index e848a2a41..6d88707f1 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -87,6 +87,7 @@ from .rule34vault import Rule34VaultCrawler
from .rule34video import Rule34VideoCrawler
from .rule34xxx import Rule34XXXCrawler
from .rule34xyz import Rule34XYZCrawler
+from .safe_soul import SafeSoulCrawler
from .saint import SaintCrawler
from .scrolller import ScrolllerCrawler
from .send_now import SendNowCrawler
diff --git a/cyberdrop_dl/crawlers/_chibisafe.py b/cyberdrop_dl/crawlers/_chibisafe.py
new file mode 100644
index 000000000..134ea0600
--- /dev/null
+++ b/cyberdrop_dl/crawlers/_chibisafe.py
@@ -0,0 +1,100 @@
+"""ChibiSafe (known as LoliSafe until v4.0.0)
+
+https://github.com/chibisafe/chibisafe
+https://chibisafe.moe/docs
+https://chibisafe.app/
+
+This is the file host framework used by bunkr, saint, cyberdrop, etc
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import datetime # noqa: TC003
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from pydantic import Field
+
+from cyberdrop_dl.crawlers.crawler import Crawler
+from cyberdrop_dl.models import AliasModel
+from cyberdrop_dl.utils.dates import to_timestamp
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, type_adapter
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.crawlers.crawler import SupportedPaths
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+
[email protected](slots=True)
+class File:
+ name: str
+ url: str
+ createdAt: datetime.datetime | None = None # noqa: N815
+ original: str | None = None
+
+
+class Album(AliasModel):
+ id: str = ""
+ name: str = Field(validation_alias="title")
+ files: list[File]
+
+
+_parse_file = type_adapter(File)
+
+
+class ChibiSafeCrawler(Crawler, is_abc=True):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Album": "/a/<album_id>",
+ "File": "/<file_id>",
+ }
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["a", album_id]:
+ return await self.album(scrape_item, album_id)
+ case [file_id]:
+ return await self.file(scrape_item, file_id)
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:
+ content: dict[str, Any] = await self.request_json(self.PRIMARY_URL / "api/file" / file_id)
+ file = _parse_file(content)
+ self._handle_file(scrape_item, file)
+
+ @error_handling_wrapper
+ async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:
+ content = await self.request_text(self.PRIMARY_URL / "api/album" / album_id)
+ album = Album.model_validate_json(content, by_alias=True)
+ album.id = album_id
+ return await self._handle_album(scrape_item, album)
+
+ async def _handle_album(self, scrape_item: ScrapeItem, album: Album) -> None:
+ title = self.create_title(album.name, album.id)
+ scrape_item.setup_as_album(title, album_id=album.id)
+ results = await self.get_album_results(album.id)
+
+ for file in album.files:
+ url = self.parse_url(file.url.removeprefix("null"))
+ if self.check_album_results(url, results):
+ continue
+ new_scrape_item = scrape_item.create_child(url)
+ self._handle_file(new_scrape_item, file)
+ scrape_item.add_children()
+
+ @error_handling_wrapper
+ def _handle_file(self, scrape_item: ScrapeItem, file: File) -> None:
+ if scrape_item.possible_datetime is None and file.createdAt:
+ scrape_item.possible_datetime = to_timestamp(file.createdAt)
+ name = file.original or file.name
+ filename, ext = self.get_filename_and_ext(name)
+ self.create_task(
+ self.handle_file(
+ scrape_item.url,
+ scrape_item,
+ name,
+ ext,
+ custom_filename=filename,
+ )
+ )
diff --git a/cyberdrop_dl/crawlers/safe_soul.py b/cyberdrop_dl/crawlers/safe_soul.py
new file mode 100644
index 000000000..5384a9a65
--- /dev/null
+++ b/cyberdrop_dl/crawlers/safe_soul.py
@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import datetime
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl.crawlers._chibisafe import Album, ChibiSafeCrawler, File
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import css
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
+
+if TYPE_CHECKING:
+ import bs4
+
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+
+class Selector:
+ ALBUM_TITLE = "#title"
+ FILE = "#table .image-container"
+ FILE_DATE = ".details .file-date"
+ FILE_NAME = ".details .name"
+ FILE_URL = "a.image"
+
+
+class SafeSoulCrawler(ChibiSafeCrawler):
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://safe.soul.lol")
+ DOMAIN: ClassVar[str] = "safe.soul.lol"
+ FOLDER_DOMAIN: ClassVar[str] = "Safe.Soul"
+
+ async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:
+ # file endpoint is disabled
+ return await self.direct_file(scrape_item)
+
+ @error_handling_wrapper
+ async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:
+ # The album endpoint is enabled, but it returns incomplete info (ex: missing date)
+ # So we scrape the HTML
+
+ soup = await self.request_soup(scrape_item.url)
+ album = Album(
+ id=album_id,
+ name=css.select_one_get_text(soup, Selector.ALBUM_TITLE),
+ files=[_parse_file(tag) for tag in soup.select(Selector.FILE)],
+ )
+ return await self._handle_album(scrape_item, album)
+
+
+def _parse_file(file_tag: bs4.Tag) -> File:
+ timestamp = int(css.select_one_get_attr(file_tag, Selector.FILE_DATE, "data-value"))
+
+ return File(
+ name=css.select_one_get_text(file_tag, Selector.FILE_NAME),
+ url=css.select_one_get_attr(file_tag, Selector.FILE_URL, "href"),
+ createdAt=datetime.datetime.fromtimestamp(timestamp),
+ )
diff --git a/tests/crawlers/test_cases/safe_soul.py b/tests/crawlers/test_cases/safe_soul.py
new file mode 100644
index 000000000..22a35756b
--- /dev/null
+++ b/tests/crawlers/test_cases/safe_soul.py
@@ -0,0 +1,44 @@
+DOMAIN = "safe.soul.lol"
+TEST_CASES = [
+ (
+ "https://safe.soul.lol/GGCW62sr.dat",
+ [
+ {
+ "url": "https://safe.soul.lol/GGCW62sr.dat",
+ "filename": "GGCW62sr.dat",
+ "referer": "https://safe.soul.lol/GGCW62sr.dat",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ "https://safe.soul.lol/a/fH8aoyxD",
+ [
+ {
+ "url": "https://safe.soul.lol/GGCW62sr.dat",
+ "filename": "GGCW62sr.dat",
+ "referer": "https://safe.soul.lol/GGCW62sr.dat",
+ "album_id": "fH8aoyxD",
+ "datetime": 1757997917,
+ "download_folder": r"re:cdl_test \(Safe\.Soul\)",
+ },
+ {
+ "url": "https://safe.soul.lol/rQPLKGeA.zip",
+ "filename": "rQPLKGeA.zip",
+ "referer": "https://safe.soul.lol/rQPLKGeA.zip",
+ "album_id": "fH8aoyxD",
+ "datetime": 1757997915,
+ "download_folder": r"re:cdl_test \(Safe\.Soul\)",
+ },
+ {
+ "url": "https://safe.soul.lol/8WXnedx1.bin",
+ "filename": "8WXnedx1.bin",
+ "referer": "https://safe.soul.lol/8WXnedx1.bin",
+ "album_id": "fH8aoyxD",
+ "datetime": 1757997914,
+ "download_folder": r"re:cdl_test \(Safe\.Soul\)",
+ },
+ ],
+ ),
+]