Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add upfile support
NTFSvolume9595c999bb19
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex a2874cc4c..70f654dc4 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -239,6 +239,15 @@ class Crawler(HTTPMixin, HLSMixin, ABC):def check_host_match(cls, host: str) -> bool:return bool(host)+ def _build_downloader[T: Downloader](self, cls: type[T] = Downloader) -> T:+ assert self.__dl_config__.server_lock is not None+ return cls(+ self.manager,+ use_server_lock=self.__dl_config__.server_lock,+ slots=self.__dl_config__.slots,+ domain=self.FOLDER_DOMAIN,+ )+@finaldef __init__(self, manager: Manager, task_mng: aio.TaskManager, tui: ScrapingUI) -> None:self.manager: Manager = manager@@ -251,13 +260,8 @@ class Crawler(HTTPMixin, HLSMixin, ABC):self.config: Config = manager.configself.client: HTTPClient = manager.http_client_CHECK_DL_CAPACITY.set(self.config.downloads.back_pressure)- assert self.__dl_config__.server_lock is not None- self.downloader: Downloader = Downloader(- manager,- use_server_lock=self.__dl_config__.server_lock,- slots=self.__dl_config__.slots,- domain=self.FOLDER_DOMAIN,- )++ self.downloader: Downloader = self._build_downloader()self.__http_ctx__: HTTPContext = HTTPContext.build(self.DOMAIN, self.__http_config__, self.__throttle)try:diff --git a/cyberdrop_dl/crawlers/upfile.py b/cyberdrop_dl/crawlers/upfile.pynew file mode 100644index 000000000..9326c9f85--- /dev/null+++ b/cyberdrop_dl/crawlers/upfile.py@@ -0,0 +1,241 @@+from __future__ import annotations++import base64+import dataclasses+import hashlib+import json+from collections import deque+from contextvars import ContextVar+from typing import TYPE_CHECKING, Any, ClassVar, Literal, override++from Crypto.Cipher import AES+from Crypto.Util import Counter++from cyberdrop_dl import aio+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths+from cyberdrop_dl.downloader.http import Downloader+from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.mediaprops import Codecs+from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem+from cyberdrop_dl.utils import css+from cyberdrop_dl.utils.dataclass import deserialize+from cyberdrop_dl.utils.errors import error_handling_wrapper+from cyberdrop_dl.utils.m3u8 import M3U8, Rendition++if TYPE_CHECKING:+ from cyberdrop_dl.url_objects import ScrapeItem++_PT = ContextVar[str]("_PT")+_MEDIA_ID = ContextVar[str]("_MEDIA_ID")+_KEY = ContextVar[bytes]("_KEY")+++class UpFileDownloader(Downloader):+ @override+ async def _download_file(self, media_item: MediaItem) -> bool | None:+ downloaded = await super()._download_file(media_item)+ if downloaded:+ key, media_key = _get_keys(media_item.real_url)+ content = await aio.read_bytes(media_item.path)+ await aio.write_bytes(media_item.path, await _decrypt(media_key, content, key))+ return downloaded+++def _get_keys(url: AbsoluteHttpURL) -> tuple[bytes, str]:+ key, media_id = _KEY.get(), _MEDIA_ID.get()+ match url.parts[-2:]:+ case [_, "vinit"]:+ media_key = f"init:{media_id}"+ case [_, "img"]:+ media_key = f"img:{media_id}"+ case [_, "thumb"]:+ media_key = f"thumb:{media_id}"+ case ["seg", idx]:+ media_key = f"seg:{media_id}:{idx}"+ case _:+ raise ValueError(f"Unknown media URL {url}")++ UpFileCrawler.get_logger().debug("Decrypting content for file %s, key = %s, media_key = %s", url, key, media_key)+ return key, media_key++[email protected]_thread+def _decrypt(media_key: str, content: bytes, key: bytes) -> bytes:+ hash_bytes = hashlib.sha256(media_key.encode("utf-8")).digest()+ iv = hash_bytes[:8].ljust(16, b"\x00")+ ctr = Counter.new(64, prefix=iv[:8], initial_value=int.from_bytes(iv[8:], "big"))+ return AES.new(key, AES.MODE_CTR, counter=ctr).decrypt(content)+++class UpFileCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Share": "/public/<path>...",+ "**NOTES**": "Does NOT support subfolders",+ }+ DOMAIN: ClassVar[str] = "upfile.top"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://upfile.top")++ def __post_init__(self) -> None:+ self.downloader: UpFileDownloader = self._build_downloader(UpFileDownloader) # pyright: ignore[reportIncompatibleVariableOverride]+ self.api: UpFileAPI = UpFileAPI.from_crawler(self)++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["public", _, *_]:+ await self.share(scrape_item, scrape_item.url.path.removeprefix("/public/"))+ case _:+ raise ValueError++ @error_handling_wrapper+ async def share(self, scrape_item: ScrapeItem, path: str) -> None:+ await self.api.boot(path)+ folder = await self.api.list(path)+ scrape_item.setup_as_album(self.create_title(folder.name, folder.id), album_id=folder.id)+ await self._walk_nodes(scrape_item, folder, path)++ async def _walk_nodes(self, scrape_item: ScrapeItem, folder: Folder, path: str) -> None:+ subfolders: deque[dict[str, Any]] = deque()+ nodes = folder.items++ while True:+ for node in nodes:+ if node["type"] == "folder":+ self.log.warning("Subfolder ignored: %s", node)+ subfolders.append(node)+ continue++ if node["type"] != "file":+ self.log.warning("Unknown node type: %s", node)+ continue++ file = File.parse(node, path=path)+ url = scrape_item.url / "m" / file.id+ new_item = scrape_item.create_child(url)+ self.create_eager_task(self._file(new_item, file))+ scrape_item.add_children()++ if not subfolders:+ return++ raise ScrapeError(422, f"Subfolders are not supported:\n {tuple(subfolders)}")+ nodes = (await self.api.list(subfolders.popleft())).items++ @error_handling_wrapper+ async def _file(self, scrape_item: ScrapeItem, file: File) -> None:+ media_url = self.api.ENTRYPOINT / file.path / "m" / file.id+ if await self.check_complete(media_url):+ return++ _MEDIA_ID.set(file.id)+ filename, ext = self.get_filename_and_ext(file.name)+ debrid_url = m3u8 = meta = None+ match file.kind:+ case "video":+ meta = await self.api.video(file.path, file.id)+ m3u8 = Rendition(_create_m3u8(media_url, len(meta.segments), meta.duration), None, None)+ case "image":+ debrid_url = media_url / "img"+ case _:+ raise ScrapeError(422, f"Unsupported file type: {file}")++ await self.handle_file(+ media_url,+ scrape_item,+ file.name,+ ext,+ custom_filename=filename,+ headers=self.api.headers,+ thumbnail=media_url / "thumb" if file.thumb else None,+ debrid_link=debrid_url,+ m3u8=m3u8,+ metadata=meta,+ )++ @override+ def _prepare_media_item(self, media_item: MediaItem) -> None:+ media_item.headers.update(self.api.headers)++ @override+ async def sniff_ext(self, thumbnail: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> str:+ if thumbnail.name == "thumb":+ return ".jpg"+ return await super().sniff_ext(thumbnail, headers)++[email protected](slots=True, frozen=True)+class File:+ id: str+ path: str+ type: Literal["file"]+ name: str+ thumb: bool+ status: str+ kind: str++ parse = classmethod(deserialize)++[email protected](slots=True, frozen=True)+class Folder:+ id: str+ name: str+ items: list[dict[str, Any]]+ type: Literal["folder"] = "folder"++[email protected](slots=True, frozen=True)+class VideoMeta:+ segments: list[int]+ codecs: Codecs+ duration: int+ width: int+ height: int+++class UpFileAPI(API):+ ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://upfile.top/api/s")++ @property+ def headers(self) -> dict[str, str]:+ return {"x-pt": _PT.get(), "cache": "no-store"}++ async def boot(self, path: str) -> None:+ url = self.PRIMARY_URL / "public" / path+ html = await self.request_text(url)+ soup = await css.asoup(html, parse_only="script")+ boot = json.loads(css.select_text(soup, "#boot"))+ _PT.set(boot["pt"])+ init = await self.request_json(self.ENTRYPOINT / path / "init", "POST", json={}, headers=self.headers)+ _KEY.set(base64.b64decode(init["key"]))++ async def list(self, path: str) -> Folder:+ url = self.ENTRYPOINT / path / "list"+ data = await self.request_json(url, headers=self.headers)+ return deserialize(Folder, data["folder"], items=data["items"])++ async def video(self, path: str, media_id: str) -> VideoMeta:+ play = self.ENTRYPOINT / path / "play"+ data = await self.request_json(play, "POST", json={"id": media_id}, headers=self.headers)+ return deserialize(VideoMeta, meta := data["meta"], codecs=Codecs.parse(meta["codecs"]))+++def _create_m3u8(url: AbsoluteHttpURL, segments: int, duration: int) -> M3U8:+ target_duration = int(duration) + (1 if duration % 1 > 0 else 0)++ def lines():+ yield "#EXTM3U"+ yield f"#EXT-X-TARGETDURATION:{target_duration}"+ yield "#EXT-X-VERSION:3"+ yield "#EXT-X-MEDIA-SEQUENCE:0"+ yield "#EXT-X-PLAYLIST-TYPE:VOD"+ yield f"#EXTINF:{duration:.6f},"+ yield "vinit"++ for idx in range(segments):+ yield f"#EXTINF:{duration:.6f},"+ yield f"seg/{idx}"++ yield "#EXT-X-ENDLIST"++ content = "\n".join(lines())+ return M3U8(content, base_uri=url, media_type="video", source=url)diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.pyindex 4e3306134..a7702cf71 100644--- a/cyberdrop_dl/downloader/http.py+++ b/cyberdrop_dl/downloader/http.py@@ -165,7 +165,7 @@ class Downloader:async def __download_w_retries(self, media_item: MediaItem) -> bool:while True:try:- return bool(await self.__download_file(media_item))+ return bool(await self._download_file(media_item))except DownloadError as e:if not self.SUPPORTS_RETRIES or not e.retry or media_item.attempts >= self.config.downloads.attempts:@@ -211,7 +211,7 @@ class Downloader:await self.capacity.notify()yield- async def __download_file(self, media_item: MediaItem) -> bool | None: # noqa: C901+ async def _download_file(self, media_item: MediaItem) -> bool | None: # noqa: C901_check_url(media_item)media_item.attempts += 1try:diff --git a/tests/crawlers/test_cases/upfile_top.py b/tests/crawlers/test_cases/upfile_top.pynew file mode 100644index 000000000..ad0367a5b--- /dev/null+++ b/tests/crawlers/test_cases/upfile_top.py@@ -0,0 +1,108 @@+DOMAIN = "upfile.top"+TEST_CASES = [+ {+ "url": "https://upfile.top/public/prfC/Autss4r8d",+ "results": [+ {+ "url": "https://upfile.top/api/s/prfC/Autss4r8d/m/2f12933dbba30d5df072fdc10ed12e1d",+ "filename": "8.jpg",+ "debrid_url": "https://upfile.top/api/s/prfC/Autss4r8d/m/2f12933dbba30d5df072fdc10ed12e1d/img",+ "original_filename": "8.jpg",+ "referer": "https://upfile.top/public/prfC/Autss4r8d/m/2f12933dbba30d5df072fdc10ed12e1d",+ "album_id": "e96f49128be6baa50395257679a84226",+ "uploaded_at": None,+ "download_folder": "re:Sarışın Güzel (Upfile.top)",+ "thumbnail": "https://upfile.top/api/s/prfC/Autss4r8d/m/2f12933dbba30d5df072fdc10ed12e1d/thumb",+ },+ {+ "url": "https://upfile.top/api/s/prfC/Autss4r8d/m/6693f1e253064e5e5e509e5c2f9fc1a6",+ "filename": "7.jpg",+ "debrid_url": "https://upfile.top/api/s/prfC/Autss4r8d/m/6693f1e253064e5e5e509e5c2f9fc1a6/img",+ "original_filename": "7.jpg",+ "referer": "https://upfile.top/public/prfC/Autss4r8d/m/6693f1e253064e5e5e509e5c2f9fc1a6",+ "album_id": "e96f49128be6baa50395257679a84226",+ "uploaded_at": None,+ "download_folder": "re:Sarışın Güzel (Upfile.top)",+ "thumbnail": "https://upfile.top/api/s/prfC/Autss4r8d/m/6693f1e253064e5e5e509e5c2f9fc1a6/thumb",+ },+ {+ "url": "https://upfile.top/api/s/prfC/Autss4r8d/m/7cac4752c559b95e47b1fc614cceeb1e",+ "filename": "9.png",+ "debrid_url": "https://upfile.top/api/s/prfC/Autss4r8d/m/7cac4752c559b95e47b1fc614cceeb1e/img",+ "original_filename": "9.png",+ "referer": "https://upfile.top/public/prfC/Autss4r8d/m/7cac4752c559b95e47b1fc614cceeb1e",+ "album_id": "e96f49128be6baa50395257679a84226",+ "uploaded_at": None,+ "download_folder": "re:Sarışın Güzel (Upfile.top)",+ "thumbnail": "https://upfile.top/api/s/prfC/Autss4r8d/m/7cac4752c559b95e47b1fc614cceeb1e/thumb",+ },+ {+ "url": "https://upfile.top/api/s/prfC/Autss4r8d/m/3c4eb3f95c56710a84d7a8f87af6167c",+ "filename": "5.mp4",+ "debrid_url": None,+ "original_filename": "5.mp4",+ "referer": "https://upfile.top/public/prfC/Autss4r8d/m/3c4eb3f95c56710a84d7a8f87af6167c",+ "album_id": "e96f49128be6baa50395257679a84226",+ "uploaded_at": None,+ "download_folder": "re:Sarışın Güzel (Upfile.top)",+ "thumbnail": "https://upfile.top/api/s/prfC/Autss4r8d/m/3c4eb3f95c56710a84d7a8f87af6167c/thumb",+ },+ {+ "url": "https://upfile.top/api/s/prfC/Autss4r8d/m/7149959390cdd458ae86c022083d4956",+ "filename": "2.mp4",+ "debrid_url": None,+ "original_filename": "2.mp4",+ "referer": "https://upfile.top/public/prfC/Autss4r8d/m/7149959390cdd458ae86c022083d4956",+ "album_id": "e96f49128be6baa50395257679a84226",+ "uploaded_at": None,+ "download_folder": "re:Sarışın Güzel (Upfile.top)",+ "thumbnail": "https://upfile.top/api/s/prfC/Autss4r8d/m/7149959390cdd458ae86c022083d4956/thumb",+ },+ {+ "url": "https://upfile.top/api/s/prfC/Autss4r8d/m/22362a8ac7250c1397a7b449e8845257",+ "filename": "6.mp4",+ "debrid_url": None,+ "original_filename": "6.mp4",+ "referer": "https://upfile.top/public/prfC/Autss4r8d/m/22362a8ac7250c1397a7b449e8845257",+ "album_id": "e96f49128be6baa50395257679a84226",+ "uploaded_at": None,+ "download_folder": "re:Sarışın Güzel (Upfile.top)",+ "thumbnail": "https://upfile.top/api/s/prfC/Autss4r8d/m/22362a8ac7250c1397a7b449e8845257/thumb",+ },+ {+ "url": "https://upfile.top/api/s/prfC/Autss4r8d/m/62616300b8e7f1bd1ad0a42aeecfa974",+ "filename": "1.mp4",+ "debrid_url": None,+ "original_filename": "1.mp4",+ "referer": "https://upfile.top/public/prfC/Autss4r8d/m/62616300b8e7f1bd1ad0a42aeecfa974",+ "album_id": "e96f49128be6baa50395257679a84226",+ "uploaded_at": None,+ "download_folder": "re:Sarışın Güzel (Upfile.top)",+ "thumbnail": "https://upfile.top/api/s/prfC/Autss4r8d/m/62616300b8e7f1bd1ad0a42aeecfa974/thumb",+ },+ {+ "url": "https://upfile.top/api/s/prfC/Autss4r8d/m/2fd3ae151c5699858a643bf103c8a967",+ "filename": "4.mp4",+ "debrid_url": None,+ "original_filename": "4.mp4",+ "referer": "https://upfile.top/public/prfC/Autss4r8d/m/2fd3ae151c5699858a643bf103c8a967",+ "album_id": "e96f49128be6baa50395257679a84226",+ "uploaded_at": None,+ "download_folder": "re:Sarışın Güzel (Upfile.top)",+ "thumbnail": "https://upfile.top/api/s/prfC/Autss4r8d/m/2fd3ae151c5699858a643bf103c8a967/thumb",+ },+ {+ "url": "https://upfile.top/api/s/prfC/Autss4r8d/m/1873dae12f8f0a538ac28ecb4321584c",+ "filename": "3.mp4",+ "debrid_url": None,+ "original_filename": "3.mp4",+ "referer": "https://upfile.top/public/prfC/Autss4r8d/m/1873dae12f8f0a538ac28ecb4321584c",+ "album_id": "e96f49128be6baa50395257679a84226",+ "uploaded_at": None,+ "download_folder": "re:Sarışın Güzel (Upfile.top)",+ "thumbnail": "https://upfile.top/api/s/prfC/Autss4r8d/m/1873dae12f8f0a538ac28ecb4321584c/thumb",+ },+ ],+ "count": 9,+ }+]