Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: remove client id requirement (imgur) (#1521)
NTFSvolume548ce9cf26e2
diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.pyindex bd7b060c0..ca4cd6709 100644--- a/cyberdrop_dl/crawlers/imgur.py+++ b/cyberdrop_dl/crawlers/imgur.py@@ -5,8 +5,9 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import LoginError, ScrapeError-from cyberdrop_dl.utils.utilities import error_handling_wrapper+from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.utils import css+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_betweenif TYPE_CHECKING:from cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -18,8 +19,8 @@ PRIMARY_URL = AbsoluteHttpURL("https://imgur.com/")class ImgurCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Album": "/a/...",- "Gallery": "/gallery/...",+ "Album": "/a/<album_id>",+ "Gallery": "/gallery/<slug>-<album_id>","Image": "/...","Direct links": "",}@@ -27,15 +28,26 @@ class ImgurCrawler(Crawler):DOMAIN: ClassVar[str] = "imgur"def __post_init__(self) -> None:- self.imgur_client_id = self.manager.config_manager.authentication_data.imgur.client_id- self.imgur_client_remaining = 12500- self.headers = {"Authorization": f"Client-ID {self.imgur_client_id}"}+ self.headers: dict[str, str] = {}@classmethod- def _json_response_check(cls, json_resp: Any) -> None:- if not isinstance(json_resp, dict) or "data" not in json_resp:- return- raise ScrapeError(json_resp["status"], json_resp["data"]["error"])+ def _json_response_check(cls, json_resp: dict[str, Any]) -> None:+ if data := json_resp.get("data"):+ raise ScrapeError(json_resp["status"], data["error"])++ async def async_startup(self) -> None:+ await self._get_client_id(self.PRIMARY_URL)++ # TODO: cache this+ @error_handling_wrapper+ async def _get_client_id(self, _) -> None:+ """Get public client id."""+ with self.disable_on_error("Unable to get client id"):+ soup = await self.request_soup(self.PRIMARY_URL)+ js_src = css.select_one_get_attr(soup, "script[src*='/desktop-assets/js/main']", "src")+ js_text = await self.request_text(self.parse_url(js_src))+ client_id = get_text_between(js_text, 'apiClientId:"', '"')+ self.headers["Authorization"] = f"Client-ID {client_id}"async def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.host == "i.imgur.com":@@ -47,20 +59,15 @@ class ImgurCrawler(Crawler):await self.image(scrape_item)async def gallery(self, scrape_item: ScrapeItem) -> None:- album_id = scrape_item.url.name.rsplit("-", 1)[-1]+ album_id = scrape_item.url.name.rpartition("-")[-1]scrape_item.url = PRIMARY_URL / "a" / album_idawait self.album(scrape_item)@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:- if not self.imgur_client_id:- msg = "No Imgur Client ID provided"- raise LoginError(msg)-- album_id = scrape_item.url.parts[-1]+ album_id = scrape_item.url.nametitle: str = ""- await self.check_imgur_credits(scrape_item)api_url = API_ENTRYPOINT / "album" / album_idjson_resp: dict[str, dict[str, Any]] = await self.request_json(api_url, headers=self.headers)@@ -75,12 +82,8 @@ class ImgurCrawler(Crawler):@error_handling_wrapperasync def image(self, scrape_item: ScrapeItem) -> None:- if not self.imgur_client_id:- msg = "No Imgur Client ID provided"- raise LoginError(msg)-image_id = scrape_item.url.parts[-1]- await self.check_imgur_credits(scrape_item)+api_url = API_ENTRYPOINT / "image" / image_idjson_resp: dict[str, dict[str, Any]] = await self.request_json(api_url, headers=self.headers)@@ -103,10 +106,3 @@ class ImgurCrawler(Crawler):new_scrape_item = scrape_item.create_child(link, possible_datetime=image_data["datetime"])await self.handle_direct_link(new_scrape_item)scrape_item.add_children()-- async def check_imgur_credits(self, _=None) -> None:- """Checks the remaining credits."""- json_resp = await self.request_json(API_ENTRYPOINT / "credits", headers=self.headers)- self.imgur_client_remaining = json_resp["data"]["ClientRemaining"]- if self.imgur_client_remaining < 100:- raise ScrapeError(429, "Imgur API rate limit reached")