Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add koofr support (#1476)

NTFSvolume7bdc3f6c9a4e
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 962681556..407932275 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -58,6 +58,7 @@ from .influencer_bitches import InfluencerBitchesCrawler
from .invision import INVISION_CRAWLERS
from .jpg5 import JPG5Crawler
from .kemono import KemonoCrawler
+from .koofr import KooFrCrawler
from .leakedzone import LeakedZoneCrawler
from .luscious import LusciousCrawler
from .masahub import MasahubCrawler
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index d33a31708..d42eeb0aa 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -497,7 +497,7 @@ class Crawler(ABC):
@final
async def check_complete_by_hash(
- self: Crawler, scrape_item: ScrapeItem | URL, hash_type: str, hash_value: str
+ self: Crawler, scrape_item: ScrapeItem | URL, hash_type: Literal["md5", "sha256"], hash_value: str
) -> bool:
"""Returns `True` if at least 1 file with this hash is recorded on the database"""
downloaded = await self.manager.db_manager.hash_table.check_hash_exists(hash_type, hash_value)
@@ -505,8 +505,7 @@ class Crawler(ABC):
url = scrape_item if isinstance(scrape_item, URL) else scrape_item.url
log(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded", 10)
self.manager.progress_manager.download_progress.add_previously_completed()
- return True
- return False
+ return downloaded
async def get_album_results(self, album_id: str) -> dict[str, int]:
"""Checks whether an album has completed given its domain and album id."""
diff --git a/cyberdrop_dl/crawlers/koofr.py b/cyberdrop_dl/crawlers/koofr.py
new file mode 100644
index 000000000..f4425187a
--- /dev/null
+++ b/cyberdrop_dl/crawlers/koofr.py
@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any, ClassVar, Literal
+
+from pydantic import dataclasses
+
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
+from cyberdrop_dl.exceptions import DownloadError, PasswordProtectedError, ScrapeError
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
+
+_APP_URL = AbsoluteHttpURL("https://app.koofr.net")
+_APP_LINKS = _APP_URL / "api/v2/public/links"
+_PRIMARY_URL = AbsoluteHttpURL("https://koofr.eu")
+_SHORT_LINK_CDN = AbsoluteHttpURL("https://k00.fr")
+
+
[email protected](slots=True, frozen=True)
+class Node:
+ name: str
+ type: Literal["file", "dir"]
+ modified: int
+ size: int
+ path: str = "/"
+ hash: str = "" # md5
+
+
+class KooFrCrawler(Crawler):
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "koofr.net", "koofr.eu", _SHORT_LINK_CDN.host
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Public Share": (
+ "/links/<content_id>",
+ f"{_SHORT_LINK_CDN}/<short_id>",
+ ),
+ }
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL
+ DOMAIN: ClassVar[str] = "koofr"
+
+ def __post_init__(self) -> None:
+ self.api = KooFrAPI(self)
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ if scrape_item.url.host == _SHORT_LINK_CDN.host:
+ return await self.follow_redirect(scrape_item)
+
+ match scrape_item.url.parts[1:]:
+ case ["links", content_id]:
+ return await self.share(scrape_item, content_id)
+ case _:
+ raise ValueError
+
+ async def _get_redirect_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ redirect = await super()._get_redirect_url(url)
+ if password := url.query.get("password"):
+ return redirect.update_query(password=password)
+ return redirect
+
+ @error_handling_wrapper
+ async def share(self, scrape_item: ScrapeItem, content_id: str) -> None:
+ path = scrape_item.url.query.get("path") or "/"
+ root = await self.api.get_info(content_id, path, scrape_item.password)
+ if root.type == "file":
+ return await self._file(scrape_item, root)
+
+ title = self.create_title(root.name, content_id)
+ scrape_item.setup_as_album(title, album_id=content_id)
+ await self._walk_folder(scrape_item, content_id, path)
+
+ @error_handling_wrapper
+ async def _walk_folder(self, scrape_item: ScrapeItem, content_id: str, path: str) -> None:
+ children = await self.api.get_children(content_id, path, scrape_item.password)
+ async with asyncio.TaskGroup() as tg:
+ for node in children:
+ if node.type == "file":
+ tg.create_task(self._file(scrape_item, node))
+ continue
+
+ url = scrape_item.url.update_query(path=node.path)
+ new_scrape_item = scrape_item.create_child(url)
+ new_scrape_item.add_to_parent_title(node.name)
+ tg.create_task(self._walk_folder(new_scrape_item, content_id, node.path))
+
+ @error_handling_wrapper
+ async def _file(self, scrape_item: ScrapeItem, file: Node) -> None:
+ content_id = scrape_item.url.name
+ link = _APP_URL / "content/links" / content_id / "files/get" / file.name
+ link = link.with_query(scrape_item.url.query).update_query(path=file.path)
+ if await self.check_complete_by_hash(link, "md5", file.hash):
+ return
+
+ filename, ext = self.get_filename_and_ext(file.name)
+ scrape_item.possible_datetime = file.modified // 1000
+ await self.handle_file(link, scrape_item, file.name, ext, custom_filename=filename)
+
+
+class KooFrAPI:
+ def __init__(self, crawler: KooFrCrawler) -> None:
+ self._crawler = crawler
+
+ async def get_info(self, content_id: str, path: str, password: str | None) -> Node:
+ password = password or ""
+ api_url = (_APP_LINKS / content_id).with_query(path=path, password=password)
+ try:
+ resp: dict[str, Any] = await self._crawler.request_json(api_url)
+ except DownloadError as e:
+ if e.status == 401:
+ msg = "Incorrect password" if password else None
+ raise PasswordProtectedError(msg) from e
+ raise
+
+ if not resp.get("isOnline"):
+ raise ScrapeError(404)
+
+ return Node(**resp["file"])
+
+ async def get_children(self, content_id: str, path: str, password: str | None) -> list[Node]:
+ password = password or ""
+ api_url = (_APP_LINKS / content_id / "bundle").with_query(path=path, password=password)
+ nodes: list[dict[str, Any]] = (await self._crawler.request_json(api_url))["files"]
+ base = path.removesuffix("/")
+ return [Node(path=f"{base}/{node['name']}", **node) for node in nodes]
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index d3e90caa3..b6375532f 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -195,7 +195,7 @@ class MediaItem:
if "mediafire" in domain:
return url.name
- if "mega.nz" in domain or "transfer.it" in domain:
+ if domain in ("mega.nz", "transfer.it", "koofr"):
return url.path_qs if not (frag := url.fragment) else f"{url.path_qs}#{frag}"
return url.path
diff --git a/tests/crawlers/test_cases/koofr.py b/tests/crawlers/test_cases/koofr.py
new file mode 100644
index 000000000..8501ac201
--- /dev/null
+++ b/tests/crawlers/test_cases/koofr.py
@@ -0,0 +1,41 @@
+DOMAIN = "koofr"
+TEST_CASES = [
+ (
+ "https://k00.fr/sngg51ex?password=756626",
+ [
+ {
+ "url": "https://app.koofr.net/content/links/b0299c24-366a-4a61-b395-dc4fa3d7781e/files/get/Cyberdrop-DL.v8.4.0.zip?password=756626&path=/",
+ "filename": "Cyberdrop-DL.v8.4.0.zip",
+ "referer": "https://app.koofr.net/links/b0299c24-366a-4a61-b395-dc4fa3d7781e?password=756626",
+ "album_id": None,
+ "datetime": 1761254288,
+ }
+ ],
+ ),
+ (
+ "https://k00.fr/b372of1c",
+ [
+ {
+ "url": "https://app.koofr.net/content/links/0a00467b-2901-4213-8d71-44fad80de82d/files/get/Cyberdrop-DL.v8.4.0.zip?path=/Cyberdrop-DL.v8.4.0.zip",
+ "filename": "Cyberdrop-DL.v8.4.0.zip",
+ "referer": "https://app.koofr.net/links/0a00467b-2901-4213-8d71-44fad80de82d",
+ "album_id": "0a00467b-2901-4213-8d71-44fad80de82d",
+ "download_folder": r"re:CDL test \(Koofr\)",
+ },
+ {
+ "url": "https://app.koofr.net/content/links/0a00467b-2901-4213-8d71-44fad80de82d/files/get/start_linux.sh?path=/subfolder/start_linux.sh",
+ "filename": "start_linux.sh",
+ "referer": "https://app.koofr.net/links/0a00467b-2901-4213-8d71-44fad80de82d?path=/subfolder",
+ "album_id": "0a00467b-2901-4213-8d71-44fad80de82d",
+ "download_folder": r"re:CDL test \(Koofr\)/subfolder",
+ },
+ {
+ "url": "https://app.koofr.net/content/links/0a00467b-2901-4213-8d71-44fad80de82d/files/get/start_windows.bat?path=/subfolder/start_windows.bat",
+ "filename": "start_windows.bat",
+ "referer": "https://app.koofr.net/links/0a00467b-2901-4213-8d71-44fad80de82d?path=/subfolder",
+ "album_id": "0a00467b-2901-4213-8d71-44fad80de82d",
+ "download_folder": r"re:CDL test \(Koofr\)/subfolder",
+ },
+ ],
+ ),
+]