Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: remove client id requirement (imgur) (#1521)

NTFSvolume548ce9cf26e2
diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.py
index bd7b060c0..ca4cd6709 100644
--- a/cyberdrop_dl/crawlers/imgur.py
+++ b/cyberdrop_dl/crawlers/imgur.py
@@ -5,8 +5,9 @@ from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.exceptions import LoginError, ScrapeError
-from cyberdrop_dl.utils.utilities import error_handling_wrapper
+from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.utils import css
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between
if TYPE_CHECKING:
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -18,8 +19,8 @@ PRIMARY_URL = AbsoluteHttpURL("https://imgur.com/")
class ImgurCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Album": "/a/...",
- "Gallery": "/gallery/...",
+ "Album": "/a/<album_id>",
+ "Gallery": "/gallery/<slug>-<album_id>",
"Image": "/...",
"Direct links": "",
}
@@ -27,15 +28,26 @@ class ImgurCrawler(Crawler):
DOMAIN: ClassVar[str] = "imgur"
def __post_init__(self) -> None:
- self.imgur_client_id = self.manager.config_manager.authentication_data.imgur.client_id
- self.imgur_client_remaining = 12500
- self.headers = {"Authorization": f"Client-ID {self.imgur_client_id}"}
+ self.headers: dict[str, str] = {}
@classmethod
- def _json_response_check(cls, json_resp: Any) -> None:
- if not isinstance(json_resp, dict) or "data" not in json_resp:
- return
- raise ScrapeError(json_resp["status"], json_resp["data"]["error"])
+ def _json_response_check(cls, json_resp: dict[str, Any]) -> None:
+ if data := json_resp.get("data"):
+ raise ScrapeError(json_resp["status"], data["error"])
+
+ async def async_startup(self) -> None:
+ await self._get_client_id(self.PRIMARY_URL)
+
+ # TODO: cache this
+ @error_handling_wrapper
+ async def _get_client_id(self, _) -> None:
+ """Get public client id."""
+ with self.disable_on_error("Unable to get client id"):
+ soup = await self.request_soup(self.PRIMARY_URL)
+ js_src = css.select_one_get_attr(soup, "script[src*='/desktop-assets/js/main']", "src")
+ js_text = await self.request_text(self.parse_url(js_src))
+ client_id = get_text_between(js_text, 'apiClientId:"', '"')
+ self.headers["Authorization"] = f"Client-ID {client_id}"
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.host == "i.imgur.com":
@@ -47,20 +59,15 @@ class ImgurCrawler(Crawler):
await self.image(scrape_item)
async def gallery(self, scrape_item: ScrapeItem) -> None:
- album_id = scrape_item.url.name.rsplit("-", 1)[-1]
+ album_id = scrape_item.url.name.rpartition("-")[-1]
scrape_item.url = PRIMARY_URL / "a" / album_id
await self.album(scrape_item)
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
- if not self.imgur_client_id:
- msg = "No Imgur Client ID provided"
- raise LoginError(msg)
-
- album_id = scrape_item.url.parts[-1]
+ album_id = scrape_item.url.name
title: str = ""
- await self.check_imgur_credits(scrape_item)
api_url = API_ENTRYPOINT / "album" / album_id
json_resp: dict[str, dict[str, Any]] = await self.request_json(api_url, headers=self.headers)
@@ -75,12 +82,8 @@ class ImgurCrawler(Crawler):
@error_handling_wrapper
async def image(self, scrape_item: ScrapeItem) -> None:
- if not self.imgur_client_id:
- msg = "No Imgur Client ID provided"
- raise LoginError(msg)
-
image_id = scrape_item.url.parts[-1]
- await self.check_imgur_credits(scrape_item)
+
api_url = API_ENTRYPOINT / "image" / image_id
json_resp: dict[str, dict[str, Any]] = await self.request_json(api_url, headers=self.headers)
@@ -103,10 +106,3 @@ class ImgurCrawler(Crawler):
new_scrape_item = scrape_item.create_child(link, possible_datetime=image_data["datetime"])
await self.handle_direct_link(new_scrape_item)
scrape_item.add_children()
-
- async def check_imgur_credits(self, _=None) -> None:
- """Checks the remaining credits."""
- json_resp = await self.request_json(API_ENTRYPOINT / "credits", headers=self.headers)
- self.imgur_client_remaining = json_resp["data"]["ClientRemaining"]
- if self.imgur_client_remaining < 100:
- raise ScrapeError(429, "Imgur API rate limit reached")