Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove reddit support (#1616)

NTFSvolume5f29ecfbc600
diff --git a/cyberdrop_dl/config/auth_model.py b/cyberdrop_dl/config/auth_model.py
index db468d453..cbc5faf1f 100755
--- a/cyberdrop_dl/config/auth_model.py
+++ b/cyberdrop_dl/config/auth_model.py
@@ -28,11 +28,6 @@ class KemonoAuth(AliasModel):
session: str = ""
-class RedditAuth(AliasModel):
- personal_use_script: str = Field("", "reddit_personal_use_script")
- secret: str = Field("", "reddit_secret")
-
-
class GoFileAuth(AliasModel):
api_key: str = Field("", "gofile_api_key")
@@ -54,4 +49,3 @@ class AuthSettings(ConfigModel):
meganz: MegaNzAuth = Field(MegaNzAuth(), "MegaNz")
pixeldrain: PixeldrainAuth = Field(PixeldrainAuth(), "PixelDrain")
realdebrid: RealDebridAuth = Field(RealDebridAuth(), "RealDebrid")
- reddit: RedditAuth = Field(RedditAuth(), "Reddit")
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index bbece60f0..6fcd75689 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -101,7 +101,6 @@ from .pornzog import PornZogCrawler
from .postimg import PostImgCrawler
from .ranoz import RanozCrawler, RootzCrawler
from .realbooru import RealBooruCrawler
-from .reddit import RedditCrawler
from .redgifs import RedGifsCrawler
from .rule34vault import Rule34VaultCrawler
from .rule34video import Rule34VideoCrawler
diff --git a/cyberdrop_dl/crawlers/reddit.py b/cyberdrop_dl/crawlers/reddit.py
deleted file mode 100644
index d78824ab5..000000000
--- a/cyberdrop_dl/crawlers/reddit.py
+++ /dev/null
@@ -1,195 +0,0 @@
-from __future__ import annotations
-
-from contextlib import contextmanager
-from typing import TYPE_CHECKING, ClassVar, NotRequired, TypedDict
-
-import asyncprawcore
-from asyncpraw import Reddit
-
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.exceptions import LoginError, NoExtensionError, ScrapeError
-from cyberdrop_dl.utils.utilities import error_handling_wrapper
-
-if TYPE_CHECKING:
- from collections.abc import AsyncIterator, Generator
-
- from asyncpraw.models import Redditor, Submission, Subreddit
-
- from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-
-
-class MediaFile(TypedDict):
- y: int # Height
- x: int # Width
-
-
-class MediaSource(MediaFile):
- u: NotRequired[str] # URL
- gif: NotRequired[str] # URL
- mp4: NotRequired[str] # URL
-
-
-class MediaMetadata(TypedDict):
- status: str
- m: str # mimetype
- p: list[MediaFile] # Previews
- o: list[MediaFile] # Originals? Options?
- s: MediaSource # Source
-
-
-PRIMARY_URL = AbsoluteHttpURL("https://www.reddit.com/")
-
-
-class RedditCrawler(Crawler):
- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "User": ("/user/<user>", "/user/<user>/...", "/u/<user>"),
- "Subreddit:": "/r/<subreddit>",
- "Direct links": "",
- }
- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "reddit", "redd.it"
- DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{title}"
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
- DOMAIN: ClassVar[str] = "reddit"
- _RATE_LIMIT = 5, 1
-
- def __post_init__(self) -> None:
- self.reddit_personal_use_script = self.manager.config_manager.authentication_data.reddit.personal_use_script
- self.reddit_secret = self.manager.config_manager.authentication_data.reddit.secret
- self.logged_in = False
-
- async def async_startup(self) -> None:
- await self.check_login(PRIMARY_URL / "login")
- if not self.logged_in:
- return
-
- self._session = self.manager.client_manager.reddit_session
- self._reddit = Reddit(
- client_id=self.reddit_personal_use_script,
- client_secret=self.reddit_secret,
- user_agent="CyberDrop-DL",
- requestor_kwargs={"session": self._session},
- check_for_updates=False,
- )
-
- async def fetch(self, scrape_item: ScrapeItem) -> None:
- if not self.logged_in:
- return
-
- async with self.client.client_manager.cache_control(self._session):
- if any(part in scrape_item.url.parts for part in ("user", "u")):
- return await self.user(scrape_item)
- if any(part in scrape_item.url.parts for part in ("comments", "r")):
- return await self.subreddit(scrape_item)
- if "redd.it" in scrape_item.url.host:
- return await self.media(scrape_item)
- raise ValueError
-
- @error_handling_wrapper
- async def user(self, scrape_item: ScrapeItem) -> None:
- username = scrape_item.url.parts[-2] if len(scrape_item.url.parts) > 3 else scrape_item.url.name
- title = self.create_title(username)
- scrape_item.setup_as_profile(title)
- user: Redditor = await self._reddit.redditor(username)
- submissions: AsyncIterator[Submission] = user.submissions.new(limit=None)
- await self.get_posts(scrape_item, submissions)
-
- @error_handling_wrapper
- async def subreddit(self, scrape_item: ScrapeItem) -> None:
- subreddit_name: str = scrape_item.url.name or scrape_item.url.parts[-2]
- title = self.create_title(subreddit_name)
- scrape_item.setup_as_profile(title)
- subreddit: Subreddit = await self._reddit.subreddit(subreddit_name)
- submissions: AsyncIterator[Submission] = subreddit.new(limit=None) # type: ignore[reportArgumentType]
- await self.get_posts(scrape_item, submissions)
-
- @error_handling_wrapper
- async def get_posts(self, scrape_item: ScrapeItem, submissions: AsyncIterator[Submission]) -> None:
- with asyncpraw_error_handle():
- async for submission in submissions:
- new_scrape_item = scrape_item.copy()
- await self.post(new_scrape_item, submission)
- scrape_item.add_children()
-
- @error_handling_wrapper
- async def post(self, scrape_item: ScrapeItem, submission: Submission) -> None:
- try:
- link_str: str = submission.media["reddit_video"]["fallback_url"]
- except (KeyError, TypeError):
- link_str = submission.url
-
- link = self.parse_url(link_str)
- date = int(str(submission.created_utc).split(".")[0])
- scrape_item.possible_datetime = date
- scrape_item.setup_as_post("")
- post_title = self.create_separate_post_title(submission.title, submission.id, date)
- scrape_item.add_to_parent_title(post_title)
- await self.process_item(scrape_item, submission, link)
-
- @error_handling_wrapper
- async def process_item(self, scrape_item: ScrapeItem, submission: Submission, link: AbsoluteHttpURL) -> None:
- if "redd.it" in link.host:
- return await self.media(scrape_item, link)
-
- new_scrape_item = scrape_item.create_child(link)
- if "gallery" in link.parts:
- return await self.gallery(new_scrape_item, submission)
- if "reddit.com" not in link.host:
- return self.handle_external_links(new_scrape_item)
-
- msg = f"found on {scrape_item.parent}"
- if scrape_item.parent != scrape_item.origin:
- msg += f" from {scrape_item.origin}"
- self.log(f"Skipping nested thread URL {link} {msg}")
-
- async def gallery(self, scrape_item: ScrapeItem, submission: Submission) -> None:
- media_metadata: dict[str, MediaMetadata] = getattr(submission, "media_metadata", {})
- if not media_metadata:
- return
-
- for item in media_metadata.values():
- if item["status"] != "valid":
- continue
- source = item["s"]
- link_str = source.get("u") or source.get("gif") or source.get("mp4")
- if not link_str:
- # TODO: Move this logic to its own method so we can raise an error on each individual link
- continue
- link = self.parse_url(link_str).with_host("i.redd.it").with_query(None)
- await self.media(scrape_item, link)
- scrape_item.add_children()
-
- @error_handling_wrapper
- async def media(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:
- """Handles media links."""
- url = link or scrape_item.url
- try:
- filename, ext = self.get_filename_and_ext(url.name)
- except NoExtensionError:
- async with self.request(url) as resp:
- if url == resp.url:
- raise
- scrape_item.url = url = resp.url
- with asyncpraw_error_handle():
- post = await self._reddit.submission(url=str(url))
-
- return await self.post(scrape_item, post)
-
- await self.handle_file(url, scrape_item, filename, ext)
-
- @error_handling_wrapper
- async def check_login(self, _) -> None:
- if not (self.reddit_personal_use_script and self.reddit_secret):
- msg = "Reddit API credentials were not provided"
- raise LoginError(message=msg)
- self.logged_in = True
-
-
-@contextmanager
-def asyncpraw_error_handle() -> Generator[None]:
- try:
- yield
- except asyncprawcore.exceptions.ResponseException as e:
- raise ScrapeError(e.response.status, message=str(e)) from None
- except asyncprawcore.exceptions.AsyncPrawcoreException as e:
- raise ScrapeError(422, message=str(e)) from None
diff --git a/cyberdrop_dl/database/__init__.py b/cyberdrop_dl/database/__init__.py
index a793aba58..4e263b15f 100644
--- a/cyberdrop_dl/database/__init__.py
+++ b/cyberdrop_dl/database/__init__.py
@@ -20,7 +20,7 @@ class Database:
async def startup(self) -> None:
"""Startup process for the DBManager."""
- self._db_conn = await aiosqlite.connect(self._db_path)
+ self._db_conn = await aiosqlite.connect(self._db_path, timeout=20)
self._db_conn.row_factory = aiosqlite.Row
self.history_table = HistoryTable(self)
self.hash_table = HashTable(self)
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 5a066f1ec..7240a0389 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -137,7 +137,7 @@ class ClientManager:
self.download_client = DownloadClient(manager, self)
self.flaresolverr = FlareSolverr(manager)
self.file_locks: WeakAsyncLocks[str] = WeakAsyncLocks()
- self.reddit_session: aiohttp.ClientSession
+
self._session: aiohttp.ClientSession
self._download_session: aiohttp.ClientSession
self._curl_session: AsyncSession[CurlResponse]
@@ -145,7 +145,6 @@ class ClientManager:
def _startup(self) -> None:
self._session = self.new_scrape_session()
- self.reddit_session = self.new_scrape_session()
self._download_session = self.new_download_session()
if _curl_import_error is not None:
return
@@ -158,7 +157,6 @@ class ClientManager:
async def __aexit__(self, *args) -> None:
await self._session.close()
- await self.reddit_session.close()
await self._download_session.close()
if _curl_import_error is not None:
return
diff --git a/docs/assets/reddit_personal_script_setup_1.png b/docs/assets/reddit_personal_script_setup_1.png
deleted file mode 100755
index f93398ada..000000000
Binary files a/docs/assets/reddit_personal_script_setup_1.png and /dev/null differ
diff --git a/docs/assets/reddit_personal_script_setup_2.png b/docs/assets/reddit_personal_script_setup_2.png
deleted file mode 100755
index d4567c1b4..000000000
Binary files a/docs/assets/reddit_personal_script_setup_2.png and /dev/null differ
diff --git a/docs/reference/configuration-options/authentication.md b/docs/reference/configuration-options/authentication.md
index 6d77675cb..b314770da 100644
--- a/docs/reference/configuration-options/authentication.md
+++ b/docs/reference/configuration-options/authentication.md
@@ -97,25 +97,6 @@ You can get your API key here (you must be logged in): [https://real-debrid.com/
<details>
-<summary>Reddit</summary>
-
-In order to scrape files from Reddit, you'll need to create an app on reddit's website (it's free): [https://www.reddit.com/prefs/apps](https://www.reddit.com/prefs/apps)
-
-Select `script` as the app type. Any name can be used. For the redirect URI you can use any website, even a fake one like `http://your_username.cyberdrop-dl`. Click `create app` to get your credentials.
-
-![reddit_personal_script_setup_1](../../assets/reddit_personal_script_setup_1.png)
-![reddit_personal_script_setup_2](../../assets/reddit_personal_script_setup_2.png)
-
-After generating the app, you need to provide Cyberdrop-DL these values:
-
-- `personal_use_script`
-
-- `secret`
-
-</details>
-
-<details>
-
<summary>Mega.nz</summary>
If you have premium or want to download files/folders only shared with you, provide CDL these values:
diff --git a/docs/reference/configuration-options/settings/download_options.md b/docs/reference/configuration-options/settings/download_options.md
index e955085df..815828f47 100644
--- a/docs/reference/configuration-options/settings/download_options.md
+++ b/docs/reference/configuration-options/settings/download_options.md
@@ -158,7 +158,6 @@ This option only works with sites that have 'posts':
- `Forums`
- `Discourse`
-- `reddit`
- `Tiktok`
- `Coomer`, `Kemono` and `Nekohouse`.
@@ -186,11 +185,10 @@ Unique Path Flags:
> `title`: post title. This is a `string`
{% hint style="warning" %}
-Not all sites support all possible flags. Ex: Posts from reddit only support the `title` flag
+Not all sites support all possible flags.
If you use a format with a field that the site does not support, CDL will replace it with `UNKNOWN_<FIELD_NAME>`
-ex: using the format `reddit post #{id}` will result in `reddit post #UNKNOWN_ID`
{% endhint %}
Setting it to `{default}` will use the default format, which is different for each crawler:
diff --git a/poetry.lock b/poetry.lock
index 6bb519914..acbe17c10 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -215,18 +215,19 @@ typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""}
[[package]]
name = "aiosqlite"
-version = "0.17.0"
+version = "0.22.1"
description = "asyncio bridge to the standard sqlite3 module"
optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.9"
groups = ["main"]
files = [
- {file = "aiosqlite-0.17.0-py3-none-any.whl", hash = "sha256:6c49dc6d3405929b1d08eeccc72306d3677503cc5e5e43771efc1e00232e8231"},
- {file = "aiosqlite-0.17.0.tar.gz", hash = "sha256:f0e6acc24bc4864149267ac82fb46dfb3be4455f99fe21df82609cc6e6baee51"},
+ {file = "aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb"},
+ {file = "aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650"},
]
-[package.dependencies]
-typing_extensions = ">=3.7.2"
+[package.extras]
+dev = ["attribution (==1.8.0)", "black (==25.11.0)", "build (>=1.2)", "coverage[toml] (==7.10.7)", "flake8 (==7.3.0)", "flake8-bugbear (==24.12.12)", "flit (==3.12.0)", "mypy (==1.19.0)", "ufmt (==2.8.0)", "usort (==1.0.8.post1)"]
+docs = ["sphinx (==8.1.3)", "sphinx-mdinclude (==0.6.2)"]
[[package]]
name = "annotated-types"
@@ -266,54 +267,6 @@ all-plugins = ["PGPy", "cryptography", "gntp", "paho-mqtt (!=2.0.*)", "smpplib"]
dev = ["babel", "coverage", "mock", "pytest", "pytest-cov", "pytest-mock", "ruff", "tox", "validate-pyproject"]
windows = ["pywin32", "tzdata"]
-[[package]]
-name = "asyncpraw"
-version = "7.8.1"
-description = "Asynchronous Python Reddit API Wrapper."
-optional = false
-python-versions = "~=3.8"
-groups = ["main"]
-files = [
- {file = "asyncpraw-7.8.1-py3-none-any.whl", hash = "sha256:8b960dd0ab404e876b8a4eb3752a3706048a9da8fb0503018b82437d973651ba"},
- {file = "asyncpraw-7.8.1.tar.gz", hash = "sha256:6fc50e3976ae106ef6190dbcca3b1b4050de7da5644adc50851ae6487679206f"},
-]
-
-[package.dependencies]
-aiofiles = "*"
-aiohttp = "<4"
-aiosqlite = "<=0.17.0"
-asyncprawcore = ">=2.4,<3"
-update_checker = ">=0.18"
-
-[package.extras]
-ci = ["coveralls"]
-dev = ["asyncpraw[lint]", "asyncpraw[test]", "packaging"]
-lint = ["asyncpraw[readthedocs]", "pre-commit", "ruff (>=0.0.292)"]
-readthedocs = ["furo", "sphinx", "sphinxcontrib-trio"]
-test = ["mock (==4.*)", "pytest (==7.*)", "pytest-asyncio (==0.18.*)", "pytest-vcr (==1.*)", "urllib3 (==1.*)", "vcrpy (==4.2.1)"]
-
-[[package]]
-name = "asyncprawcore"
-version = "2.4.0"
-description = "Low-level asynchronous communication layer for Async PRAW 7+."
-optional = false
-python-versions = "~=3.8"
-groups = ["main"]
-files = [
- {file = "asyncprawcore-2.4.0-py3-none-any.whl", hash = "sha256:3fe0f85d783ad0fd2a5d68d219b576bdbe04260ad7195c8a035d8724f57cfa45"},
- {file = "asyncprawcore-2.4.0.tar.gz", hash = "sha256:3a3359e5cd1ebe61d544a09d4b5eca7b16edfd17de07081a8415fc794f7bb62e"},
-]
-
-[package.dependencies]
-aiohttp = "<4"
-yarl = "*"
-
-[package.extras]
-ci = ["coveralls"]
-dev = ["asyncprawcore[lint]", "asyncprawcore[test]", "packaging"]
-lint = ["pre-commit", "ruff (==0.1.*)"]
-test = ["mock (==4.*)", "pytest (==7.*)", "pytest-asyncio (==0.18.*)", "pytest-vcr (==1.*)", "urllib3 (==1.*)", "vcrpy (==4.2.1)"]
-
[[package]]
name = "attrs"
version = "25.4.0"
@@ -2765,26 +2718,6 @@ tzdata = {version = "*", markers = "platform_system == \"Windows\""}
[package.extras]
devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "zest.releaser"]
-[[package]]
-name = "update-checker"
-version = "0.18.0"
-description = "A python module that will check for package updates."
-optional = false
-python-versions = "*"
-groups = ["main"]
-files = [
- {file = "update_checker-0.18.0-py3-none-any.whl", hash = "sha256:cbba64760a36fe2640d80d85306e8fe82b6816659190993b7bdabadee4d4bbfd"},
- {file = "update_checker-0.18.0.tar.gz", hash = "sha256:6a2d45bb4ac585884a6b03f9eade9161cedd9e8111545141e9aa9058932acb13"},
-]
-
-[package.dependencies]
-requests = ">=2.3.0"
-
-[package.extras]
-dev = ["black", "flake8", "pytest (>=2.7.3)"]
-lint = ["black", "flake8"]
-test = ["pytest (>=2.7.3)"]
-
[[package]]
name = "urllib3"
version = "2.6.3"
@@ -3134,4 +3067,4 @@ propcache = ">=0.2.1"
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<4"
-content-hash = "545f4d359a61658ab8960d3303e2586c9b49dc8c7dc0a0561cd197d298fe1156"
+content-hash = "3e29a6a0f9cea39203d65bca7ebafff49758de494c8fa3d59386b1f3513e645e"
diff --git a/pyproject.toml b/pyproject.toml
index 8483ccc83..72fd3cd65 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,8 +6,7 @@ dependencies = [
"aiofiles >=25.1.0",
"aiohttp[speedups] >=3.13.3",
"aiolimiter >=1.2.1",
- "aiosqlite==0.17.0",
- "asyncpraw >=7.8.1",
+ "aiosqlite>=0.22.1",
"beautifulsoup4 >=4.14.3",
"certifi >=2026.1.4",
"curl-cffi >=0.13,<0.14; implementation_name == 'cpython' ",
@@ -36,7 +35,7 @@ license = "GPL-3.0-only"
license-files = ["LICENSE"]
readme = "README.md"
requires-python = ">=3.11,<4"
-version = "8.10.0"
+version = "9.0.0"
[project.scripts]
cyberdrop-dl = "cyberdrop_dl.main:main"