Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove reddit support (#1616)
NTFSvolume5f29ecfbc600
diff --git a/cyberdrop_dl/config/auth_model.py b/cyberdrop_dl/config/auth_model.pyindex db468d453..cbc5faf1f 100755--- a/cyberdrop_dl/config/auth_model.py+++ b/cyberdrop_dl/config/auth_model.py@@ -28,11 +28,6 @@ class KemonoAuth(AliasModel):session: str = ""-class RedditAuth(AliasModel):- personal_use_script: str = Field("", "reddit_personal_use_script")- secret: str = Field("", "reddit_secret")--class GoFileAuth(AliasModel):api_key: str = Field("", "gofile_api_key")@@ -54,4 +49,3 @@ class AuthSettings(ConfigModel):meganz: MegaNzAuth = Field(MegaNzAuth(), "MegaNz")pixeldrain: PixeldrainAuth = Field(PixeldrainAuth(), "PixelDrain")realdebrid: RealDebridAuth = Field(RealDebridAuth(), "RealDebrid")- reddit: RedditAuth = Field(RedditAuth(), "Reddit")diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex bbece60f0..6fcd75689 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -101,7 +101,6 @@ from .pornzog import PornZogCrawlerfrom .postimg import PostImgCrawlerfrom .ranoz import RanozCrawler, RootzCrawlerfrom .realbooru import RealBooruCrawler-from .reddit import RedditCrawlerfrom .redgifs import RedGifsCrawlerfrom .rule34vault import Rule34VaultCrawlerfrom .rule34video import Rule34VideoCrawlerdiff --git a/cyberdrop_dl/crawlers/reddit.py b/cyberdrop_dl/crawlers/reddit.pydeleted file mode 100644index d78824ab5..000000000--- a/cyberdrop_dl/crawlers/reddit.py+++ /dev/null@@ -1,195 +0,0 @@-from __future__ import annotations--from contextlib import contextmanager-from typing import TYPE_CHECKING, ClassVar, NotRequired, TypedDict--import asyncprawcore-from asyncpraw import Reddit--from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import LoginError, NoExtensionError, ScrapeError-from cyberdrop_dl.utils.utilities import error_handling_wrapper--if TYPE_CHECKING:- from collections.abc import AsyncIterator, Generator-- from asyncpraw.models import Redditor, Submission, Subreddit-- from cyberdrop_dl.data_structures.url_objects import ScrapeItem---class MediaFile(TypedDict):- y: int # Height- x: int # Width---class MediaSource(MediaFile):- u: NotRequired[str] # URL- gif: NotRequired[str] # URL- mp4: NotRequired[str] # URL---class MediaMetadata(TypedDict):- status: str- m: str # mimetype- p: list[MediaFile] # Previews- o: list[MediaFile] # Originals? Options?- s: MediaSource # Source---PRIMARY_URL = AbsoluteHttpURL("https://www.reddit.com/")---class RedditCrawler(Crawler):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "User": ("/user/<user>", "/user/<user>/...", "/u/<user>"),- "Subreddit:": "/r/<subreddit>",- "Direct links": "",- }- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "reddit", "redd.it"- DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{title}"- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL- DOMAIN: ClassVar[str] = "reddit"- _RATE_LIMIT = 5, 1-- def __post_init__(self) -> None:- self.reddit_personal_use_script = self.manager.config_manager.authentication_data.reddit.personal_use_script- self.reddit_secret = self.manager.config_manager.authentication_data.reddit.secret- self.logged_in = False-- async def async_startup(self) -> None:- await self.check_login(PRIMARY_URL / "login")- if not self.logged_in:- return-- self._session = self.manager.client_manager.reddit_session- self._reddit = Reddit(- client_id=self.reddit_personal_use_script,- client_secret=self.reddit_secret,- user_agent="CyberDrop-DL",- requestor_kwargs={"session": self._session},- check_for_updates=False,- )-- async def fetch(self, scrape_item: ScrapeItem) -> None:- if not self.logged_in:- return-- async with self.client.client_manager.cache_control(self._session):- if any(part in scrape_item.url.parts for part in ("user", "u")):- return await self.user(scrape_item)- if any(part in scrape_item.url.parts for part in ("comments", "r")):- return await self.subreddit(scrape_item)- if "redd.it" in scrape_item.url.host:- return await self.media(scrape_item)- raise ValueError-- @error_handling_wrapper- async def user(self, scrape_item: ScrapeItem) -> None:- username = scrape_item.url.parts[-2] if len(scrape_item.url.parts) > 3 else scrape_item.url.name- title = self.create_title(username)- scrape_item.setup_as_profile(title)- user: Redditor = await self._reddit.redditor(username)- submissions: AsyncIterator[Submission] = user.submissions.new(limit=None)- await self.get_posts(scrape_item, submissions)-- @error_handling_wrapper- async def subreddit(self, scrape_item: ScrapeItem) -> None:- subreddit_name: str = scrape_item.url.name or scrape_item.url.parts[-2]- title = self.create_title(subreddit_name)- scrape_item.setup_as_profile(title)- subreddit: Subreddit = await self._reddit.subreddit(subreddit_name)- submissions: AsyncIterator[Submission] = subreddit.new(limit=None) # type: ignore[reportArgumentType]- await self.get_posts(scrape_item, submissions)-- @error_handling_wrapper- async def get_posts(self, scrape_item: ScrapeItem, submissions: AsyncIterator[Submission]) -> None:- with asyncpraw_error_handle():- async for submission in submissions:- new_scrape_item = scrape_item.copy()- await self.post(new_scrape_item, submission)- scrape_item.add_children()-- @error_handling_wrapper- async def post(self, scrape_item: ScrapeItem, submission: Submission) -> None:- try:- link_str: str = submission.media["reddit_video"]["fallback_url"]- except (KeyError, TypeError):- link_str = submission.url-- link = self.parse_url(link_str)- date = int(str(submission.created_utc).split(".")[0])- scrape_item.possible_datetime = date- scrape_item.setup_as_post("")- post_title = self.create_separate_post_title(submission.title, submission.id, date)- scrape_item.add_to_parent_title(post_title)- await self.process_item(scrape_item, submission, link)-- @error_handling_wrapper- async def process_item(self, scrape_item: ScrapeItem, submission: Submission, link: AbsoluteHttpURL) -> None:- if "redd.it" in link.host:- return await self.media(scrape_item, link)-- new_scrape_item = scrape_item.create_child(link)- if "gallery" in link.parts:- return await self.gallery(new_scrape_item, submission)- if "reddit.com" not in link.host:- return self.handle_external_links(new_scrape_item)-- msg = f"found on {scrape_item.parent}"- if scrape_item.parent != scrape_item.origin:- msg += f" from {scrape_item.origin}"- self.log(f"Skipping nested thread URL {link} {msg}")-- async def gallery(self, scrape_item: ScrapeItem, submission: Submission) -> None:- media_metadata: dict[str, MediaMetadata] = getattr(submission, "media_metadata", {})- if not media_metadata:- return-- for item in media_metadata.values():- if item["status"] != "valid":- continue- source = item["s"]- link_str = source.get("u") or source.get("gif") or source.get("mp4")- if not link_str:- # TODO: Move this logic to its own method so we can raise an error on each individual link- continue- link = self.parse_url(link_str).with_host("i.redd.it").with_query(None)- await self.media(scrape_item, link)- scrape_item.add_children()-- @error_handling_wrapper- async def media(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:- """Handles media links."""- url = link or scrape_item.url- try:- filename, ext = self.get_filename_and_ext(url.name)- except NoExtensionError:- async with self.request(url) as resp:- if url == resp.url:- raise- scrape_item.url = url = resp.url- with asyncpraw_error_handle():- post = await self._reddit.submission(url=str(url))-- return await self.post(scrape_item, post)-- await self.handle_file(url, scrape_item, filename, ext)-- @error_handling_wrapper- async def check_login(self, _) -> None:- if not (self.reddit_personal_use_script and self.reddit_secret):- msg = "Reddit API credentials were not provided"- raise LoginError(message=msg)- self.logged_in = True---@contextmanager-def asyncpraw_error_handle() -> Generator[None]:- try:- yield- except asyncprawcore.exceptions.ResponseException as e:- raise ScrapeError(e.response.status, message=str(e)) from None- except asyncprawcore.exceptions.AsyncPrawcoreException as e:- raise ScrapeError(422, message=str(e)) from Nonediff --git a/cyberdrop_dl/database/__init__.py b/cyberdrop_dl/database/__init__.pyindex a793aba58..4e263b15f 100644--- a/cyberdrop_dl/database/__init__.py+++ b/cyberdrop_dl/database/__init__.py@@ -20,7 +20,7 @@ class Database:async def startup(self) -> None:"""Startup process for the DBManager."""- self._db_conn = await aiosqlite.connect(self._db_path)+ self._db_conn = await aiosqlite.connect(self._db_path, timeout=20)self._db_conn.row_factory = aiosqlite.Rowself.history_table = HistoryTable(self)self.hash_table = HashTable(self)diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 5a066f1ec..7240a0389 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -137,7 +137,7 @@ class ClientManager:self.download_client = DownloadClient(manager, self)self.flaresolverr = FlareSolverr(manager)self.file_locks: WeakAsyncLocks[str] = WeakAsyncLocks()- self.reddit_session: aiohttp.ClientSession+self._session: aiohttp.ClientSessionself._download_session: aiohttp.ClientSessionself._curl_session: AsyncSession[CurlResponse]@@ -145,7 +145,6 @@ class ClientManager:def _startup(self) -> None:self._session = self.new_scrape_session()- self.reddit_session = self.new_scrape_session()self._download_session = self.new_download_session()if _curl_import_error is not None:return@@ -158,7 +157,6 @@ class ClientManager:async def __aexit__(self, *args) -> None:await self._session.close()- await self.reddit_session.close()await self._download_session.close()if _curl_import_error is not None:returndiff --git a/docs/assets/reddit_personal_script_setup_1.png b/docs/assets/reddit_personal_script_setup_1.pngdeleted file mode 100755index f93398ada..000000000Binary files a/docs/assets/reddit_personal_script_setup_1.png and /dev/null differdiff --git a/docs/assets/reddit_personal_script_setup_2.png b/docs/assets/reddit_personal_script_setup_2.pngdeleted file mode 100755index d4567c1b4..000000000Binary files a/docs/assets/reddit_personal_script_setup_2.png and /dev/null differdiff --git a/docs/reference/configuration-options/authentication.md b/docs/reference/configuration-options/authentication.mdindex 6d77675cb..b314770da 100644--- a/docs/reference/configuration-options/authentication.md+++ b/docs/reference/configuration-options/authentication.md@@ -97,25 +97,6 @@ You can get your API key here (you must be logged in): [https://real-debrid.com/<details>-<summary>Reddit</summary>--In order to scrape files from Reddit, you'll need to create an app on reddit's website (it's free): [https://www.reddit.com/prefs/apps](https://www.reddit.com/prefs/apps)--Select `script` as the app type. Any name can be used. For the redirect URI you can use any website, even a fake one like `http://your_username.cyberdrop-dl`. Click `create app` to get your credentials.-----After generating the app, you need to provide Cyberdrop-DL these values:--- `personal_use_script`--- `secret`--</details>--<details>-<summary>Mega.nz</summary>If you have premium or want to download files/folders only shared with you, provide CDL these values:diff --git a/docs/reference/configuration-options/settings/download_options.md b/docs/reference/configuration-options/settings/download_options.mdindex e955085df..815828f47 100644--- a/docs/reference/configuration-options/settings/download_options.md+++ b/docs/reference/configuration-options/settings/download_options.md@@ -158,7 +158,6 @@ This option only works with sites that have 'posts':- `Forums`- `Discourse`-- `reddit`- `Tiktok`- `Coomer`, `Kemono` and `Nekohouse`.@@ -186,11 +185,10 @@ Unique Path Flags:> `title`: post title. This is a `string`{% hint style="warning" %}-Not all sites support all possible flags. Ex: Posts from reddit only support the `title` flag+Not all sites support all possible flags.If you use a format with a field that the site does not support, CDL will replace it with `UNKNOWN_<FIELD_NAME>`-ex: using the format `reddit post #{id}` will result in `reddit post #UNKNOWN_ID`{% endhint %}Setting it to `{default}` will use the default format, which is different for each crawler:diff --git a/poetry.lock b/poetry.lockindex 6bb519914..acbe17c10 100644--- a/poetry.lock+++ b/poetry.lock@@ -215,18 +215,19 @@ typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""}[[package]]name = "aiosqlite"-version = "0.17.0"+version = "0.22.1"description = "asyncio bridge to the standard sqlite3 module"optional = false-python-versions = ">=3.6"+python-versions = ">=3.9"groups = ["main"]files = [- {file = "aiosqlite-0.17.0-py3-none-any.whl", hash = "sha256:6c49dc6d3405929b1d08eeccc72306d3677503cc5e5e43771efc1e00232e8231"},- {file = "aiosqlite-0.17.0.tar.gz", hash = "sha256:f0e6acc24bc4864149267ac82fb46dfb3be4455f99fe21df82609cc6e6baee51"},+ {file = "aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb"},+ {file = "aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650"},]-[package.dependencies]-typing_extensions = ">=3.7.2"+[package.extras]+dev = ["attribution (==1.8.0)", "black (==25.11.0)", "build (>=1.2)", "coverage[toml] (==7.10.7)", "flake8 (==7.3.0)", "flake8-bugbear (==24.12.12)", "flit (==3.12.0)", "mypy (==1.19.0)", "ufmt (==2.8.0)", "usort (==1.0.8.post1)"]+docs = ["sphinx (==8.1.3)", "sphinx-mdinclude (==0.6.2)"][[package]]name = "annotated-types"@@ -266,54 +267,6 @@ all-plugins = ["PGPy", "cryptography", "gntp", "paho-mqtt (!=2.0.*)", "smpplib"]dev = ["babel", "coverage", "mock", "pytest", "pytest-cov", "pytest-mock", "ruff", "tox", "validate-pyproject"]windows = ["pywin32", "tzdata"]-[[package]]-name = "asyncpraw"-version = "7.8.1"-description = "Asynchronous Python Reddit API Wrapper."-optional = false-python-versions = "~=3.8"-groups = ["main"]-files = [- {file = "asyncpraw-7.8.1-py3-none-any.whl", hash = "sha256:8b960dd0ab404e876b8a4eb3752a3706048a9da8fb0503018b82437d973651ba"},- {file = "asyncpraw-7.8.1.tar.gz", hash = "sha256:6fc50e3976ae106ef6190dbcca3b1b4050de7da5644adc50851ae6487679206f"},-]--[package.dependencies]-aiofiles = "*"-aiohttp = "<4"-aiosqlite = "<=0.17.0"-asyncprawcore = ">=2.4,<3"-update_checker = ">=0.18"--[package.extras]-ci = ["coveralls"]-dev = ["asyncpraw[lint]", "asyncpraw[test]", "packaging"]-lint = ["asyncpraw[readthedocs]", "pre-commit", "ruff (>=0.0.292)"]-readthedocs = ["furo", "sphinx", "sphinxcontrib-trio"]-test = ["mock (==4.*)", "pytest (==7.*)", "pytest-asyncio (==0.18.*)", "pytest-vcr (==1.*)", "urllib3 (==1.*)", "vcrpy (==4.2.1)"]--[[package]]-name = "asyncprawcore"-version = "2.4.0"-description = "Low-level asynchronous communication layer for Async PRAW 7+."-optional = false-python-versions = "~=3.8"-groups = ["main"]-files = [- {file = "asyncprawcore-2.4.0-py3-none-any.whl", hash = "sha256:3fe0f85d783ad0fd2a5d68d219b576bdbe04260ad7195c8a035d8724f57cfa45"},- {file = "asyncprawcore-2.4.0.tar.gz", hash = "sha256:3a3359e5cd1ebe61d544a09d4b5eca7b16edfd17de07081a8415fc794f7bb62e"},-]--[package.dependencies]-aiohttp = "<4"-yarl = "*"--[package.extras]-ci = ["coveralls"]-dev = ["asyncprawcore[lint]", "asyncprawcore[test]", "packaging"]-lint = ["pre-commit", "ruff (==0.1.*)"]-test = ["mock (==4.*)", "pytest (==7.*)", "pytest-asyncio (==0.18.*)", "pytest-vcr (==1.*)", "urllib3 (==1.*)", "vcrpy (==4.2.1)"]-[[package]]name = "attrs"version = "25.4.0"@@ -2765,26 +2718,6 @@ tzdata = {version = "*", markers = "platform_system == \"Windows\""}[package.extras]devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "zest.releaser"]-[[package]]-name = "update-checker"-version = "0.18.0"-description = "A python module that will check for package updates."-optional = false-python-versions = "*"-groups = ["main"]-files = [- {file = "update_checker-0.18.0-py3-none-any.whl", hash = "sha256:cbba64760a36fe2640d80d85306e8fe82b6816659190993b7bdabadee4d4bbfd"},- {file = "update_checker-0.18.0.tar.gz", hash = "sha256:6a2d45bb4ac585884a6b03f9eade9161cedd9e8111545141e9aa9058932acb13"},-]--[package.dependencies]-requests = ">=2.3.0"--[package.extras]-dev = ["black", "flake8", "pytest (>=2.7.3)"]-lint = ["black", "flake8"]-test = ["pytest (>=2.7.3)"]-[[package]]name = "urllib3"version = "2.6.3"@@ -3134,4 +3067,4 @@ propcache = ">=0.2.1"[metadata]lock-version = "2.1"python-versions = ">=3.11,<4"-content-hash = "545f4d359a61658ab8960d3303e2586c9b49dc8c7dc0a0561cd197d298fe1156"+content-hash = "3e29a6a0f9cea39203d65bca7ebafff49758de494c8fa3d59386b1f3513e645e"diff --git a/pyproject.toml b/pyproject.tomlindex 8483ccc83..72fd3cd65 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -6,8 +6,7 @@ dependencies = ["aiofiles >=25.1.0","aiohttp[speedups] >=3.13.3","aiolimiter >=1.2.1",- "aiosqlite==0.17.0",- "asyncpraw >=7.8.1",+ "aiosqlite>=0.22.1","beautifulsoup4 >=4.14.3","certifi >=2026.1.4","curl-cffi >=0.13,<0.14; implementation_name == 'cpython' ",@@ -36,7 +35,7 @@ license = "GPL-3.0-only"license-files = ["LICENSE"]readme = "README.md"requires-python = ">=3.11,<4"-version = "8.10.0"+version = "9.0.0"[project.scripts]cyberdrop-dl = "cyberdrop_dl.main:main"