Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add koofr support (#1476)
NTFSvolume7bdc3f6c9a4e
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex 962681556..407932275 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -58,6 +58,7 @@ from .influencer_bitches import InfluencerBitchesCrawlerfrom .invision import INVISION_CRAWLERSfrom .jpg5 import JPG5Crawlerfrom .kemono import KemonoCrawler+from .koofr import KooFrCrawlerfrom .leakedzone import LeakedZoneCrawlerfrom .luscious import LusciousCrawlerfrom .masahub import MasahubCrawlerdiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex d33a31708..d42eeb0aa 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -497,7 +497,7 @@ class Crawler(ABC):@finalasync def check_complete_by_hash(- self: Crawler, scrape_item: ScrapeItem | URL, hash_type: str, hash_value: str+ self: Crawler, scrape_item: ScrapeItem | URL, hash_type: Literal["md5", "sha256"], hash_value: str) -> bool:"""Returns `True` if at least 1 file with this hash is recorded on the database"""downloaded = await self.manager.db_manager.hash_table.check_hash_exists(hash_type, hash_value)@@ -505,8 +505,7 @@ class Crawler(ABC):url = scrape_item if isinstance(scrape_item, URL) else scrape_item.urllog(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded", 10)self.manager.progress_manager.download_progress.add_previously_completed()- return True- return False+ return downloadedasync def get_album_results(self, album_id: str) -> dict[str, int]:"""Checks whether an album has completed given its domain and album id."""diff --git a/cyberdrop_dl/crawlers/koofr.py b/cyberdrop_dl/crawlers/koofr.pynew file mode 100644index 000000000..f4425187a--- /dev/null+++ b/cyberdrop_dl/crawlers/koofr.py@@ -0,0 +1,122 @@+from __future__ import annotations++import asyncio+from typing import Any, ClassVar, Literal++from pydantic import dataclasses++from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem+from cyberdrop_dl.exceptions import DownloadError, PasswordProtectedError, ScrapeError+from cyberdrop_dl.utils.utilities import error_handling_wrapper++_APP_URL = AbsoluteHttpURL("https://app.koofr.net")+_APP_LINKS = _APP_URL / "api/v2/public/links"+_PRIMARY_URL = AbsoluteHttpURL("https://koofr.eu")+_SHORT_LINK_CDN = AbsoluteHttpURL("https://k00.fr")++[email protected](slots=True, frozen=True)+class Node:+ name: str+ type: Literal["file", "dir"]+ modified: int+ size: int+ path: str = "/"+ hash: str = "" # md5+++class KooFrCrawler(Crawler):+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "koofr.net", "koofr.eu", _SHORT_LINK_CDN.host+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Public Share": (+ "/links/<content_id>",+ f"{_SHORT_LINK_CDN}/<short_id>",+ ),+ }+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL+ DOMAIN: ClassVar[str] = "koofr"++ def __post_init__(self) -> None:+ self.api = KooFrAPI(self)++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ if scrape_item.url.host == _SHORT_LINK_CDN.host:+ return await self.follow_redirect(scrape_item)++ match scrape_item.url.parts[1:]:+ case ["links", content_id]:+ return await self.share(scrape_item, content_id)+ case _:+ raise ValueError++ async def _get_redirect_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ redirect = await super()._get_redirect_url(url)+ if password := url.query.get("password"):+ return redirect.update_query(password=password)+ return redirect++ @error_handling_wrapper+ async def share(self, scrape_item: ScrapeItem, content_id: str) -> None:+ path = scrape_item.url.query.get("path") or "/"+ root = await self.api.get_info(content_id, path, scrape_item.password)+ if root.type == "file":+ return await self._file(scrape_item, root)++ title = self.create_title(root.name, content_id)+ scrape_item.setup_as_album(title, album_id=content_id)+ await self._walk_folder(scrape_item, content_id, path)++ @error_handling_wrapper+ async def _walk_folder(self, scrape_item: ScrapeItem, content_id: str, path: str) -> None:+ children = await self.api.get_children(content_id, path, scrape_item.password)+ async with asyncio.TaskGroup() as tg:+ for node in children:+ if node.type == "file":+ tg.create_task(self._file(scrape_item, node))+ continue++ url = scrape_item.url.update_query(path=node.path)+ new_scrape_item = scrape_item.create_child(url)+ new_scrape_item.add_to_parent_title(node.name)+ tg.create_task(self._walk_folder(new_scrape_item, content_id, node.path))++ @error_handling_wrapper+ async def _file(self, scrape_item: ScrapeItem, file: Node) -> None:+ content_id = scrape_item.url.name+ link = _APP_URL / "content/links" / content_id / "files/get" / file.name+ link = link.with_query(scrape_item.url.query).update_query(path=file.path)+ if await self.check_complete_by_hash(link, "md5", file.hash):+ return++ filename, ext = self.get_filename_and_ext(file.name)+ scrape_item.possible_datetime = file.modified // 1000+ await self.handle_file(link, scrape_item, file.name, ext, custom_filename=filename)+++class KooFrAPI:+ def __init__(self, crawler: KooFrCrawler) -> None:+ self._crawler = crawler++ async def get_info(self, content_id: str, path: str, password: str | None) -> Node:+ password = password or ""+ api_url = (_APP_LINKS / content_id).with_query(path=path, password=password)+ try:+ resp: dict[str, Any] = await self._crawler.request_json(api_url)+ except DownloadError as e:+ if e.status == 401:+ msg = "Incorrect password" if password else None+ raise PasswordProtectedError(msg) from e+ raise++ if not resp.get("isOnline"):+ raise ScrapeError(404)++ return Node(**resp["file"])++ async def get_children(self, content_id: str, path: str, password: str | None) -> list[Node]:+ password = password or ""+ api_url = (_APP_LINKS / content_id / "bundle").with_query(path=path, password=password)+ nodes: list[dict[str, Any]] = (await self._crawler.request_json(api_url))["files"]+ base = path.removesuffix("/")+ return [Node(path=f"{base}/{node['name']}", **node) for node in nodes]diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex d3e90caa3..b6375532f 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -195,7 +195,7 @@ class MediaItem:if "mediafire" in domain:return url.name- if "mega.nz" in domain or "transfer.it" in domain:+ if domain in ("mega.nz", "transfer.it", "koofr"):return url.path_qs if not (frag := url.fragment) else f"{url.path_qs}#{frag}"return url.pathdiff --git a/tests/crawlers/test_cases/koofr.py b/tests/crawlers/test_cases/koofr.pynew file mode 100644index 000000000..8501ac201--- /dev/null+++ b/tests/crawlers/test_cases/koofr.py@@ -0,0 +1,41 @@+DOMAIN = "koofr"+TEST_CASES = [+ (+ "https://k00.fr/sngg51ex?password=756626",+ [+ {+ "url": "https://app.koofr.net/content/links/b0299c24-366a-4a61-b395-dc4fa3d7781e/files/get/Cyberdrop-DL.v8.4.0.zip?password=756626&path=/",+ "filename": "Cyberdrop-DL.v8.4.0.zip",+ "referer": "https://app.koofr.net/links/b0299c24-366a-4a61-b395-dc4fa3d7781e?password=756626",+ "album_id": None,+ "datetime": 1761254288,+ }+ ],+ ),+ (+ "https://k00.fr/b372of1c",+ [+ {+ "url": "https://app.koofr.net/content/links/0a00467b-2901-4213-8d71-44fad80de82d/files/get/Cyberdrop-DL.v8.4.0.zip?path=/Cyberdrop-DL.v8.4.0.zip",+ "filename": "Cyberdrop-DL.v8.4.0.zip",+ "referer": "https://app.koofr.net/links/0a00467b-2901-4213-8d71-44fad80de82d",+ "album_id": "0a00467b-2901-4213-8d71-44fad80de82d",+ "download_folder": r"re:CDL test \(Koofr\)",+ },+ {+ "url": "https://app.koofr.net/content/links/0a00467b-2901-4213-8d71-44fad80de82d/files/get/start_linux.sh?path=/subfolder/start_linux.sh",+ "filename": "start_linux.sh",+ "referer": "https://app.koofr.net/links/0a00467b-2901-4213-8d71-44fad80de82d?path=/subfolder",+ "album_id": "0a00467b-2901-4213-8d71-44fad80de82d",+ "download_folder": r"re:CDL test \(Koofr\)/subfolder",+ },+ {+ "url": "https://app.koofr.net/content/links/0a00467b-2901-4213-8d71-44fad80de82d/files/get/start_windows.bat?path=/subfolder/start_windows.bat",+ "filename": "start_windows.bat",+ "referer": "https://app.koofr.net/links/0a00467b-2901-4213-8d71-44fad80de82d?path=/subfolder",+ "album_id": "0a00467b-2901-4213-8d71-44fad80de82d",+ "download_folder": r"re:CDL test \(Koofr\)/subfolder",+ },+ ],+ ),+]