Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: improve error messages (#1270)
NTFSvolume15f0d418d6d2
diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.pyindex 04dbc8f26..d2c730cad 100644--- a/cyberdrop_dl/exceptions.py+++ b/cyberdrop_dl/exceptions.py@@ -1,5 +1,6 @@from __future__ import annotations+import dataclassesfrom http import HTTPStatusfrom pathlib import Pathfrom typing import TYPE_CHECKING@@ -20,7 +21,7 @@ def _format_error(ui_failure: str, message: str) -> str:# See: https://developers.cloudflare.com/support/troubleshooting/cloudflare-errors/troubleshooting-cloudflare-5xx-errors/-CLOUDFLARE_ERRORS = {+CLOUDFLARE_HTTP_ERROR_CODES = {520: "Unexpected Response",521: "Web Server Down",522: "Connection Timeout",@@ -31,6 +32,19 @@ CLOUDFLARE_ERRORS = {530: "IP Banned / Restricted",}+# https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#Nonstandard_codes+# Not all of them, just the ones we actually expect to see and do not shadow standard onces+NON_STANDARD_HTTP_ERROR_CODES = {+ 419: "Page Expired",+ 509: "Bandwidth Limit Exceeded",+}++HTTP_ERROR_CODES = {+ **NON_STANDARD_HTTP_ERROR_CODES,+ **CLOUDFLARE_HTTP_ERROR_CODES,+ **{code.value: code.phrase for code in HTTPStatus},+}+class TooManyCrawlerErrors(Exception): # noqa: N818"""This exception will be raised after a crawler had too many errors proccesing URLs"""@@ -219,14 +233,18 @@ class InvalidYamlError(CDLBaseError):def create_error_msg(error: int | str) -> str:if isinstance(error, str):return error- try:- msg = HTTPStatus(error).phrase- return f"{error} {msg}"- except ValueError:- cloudflare_error = CLOUDFLARE_ERRORS.get(error)- if cloudflare_error:- return f"{error} {cloudflare_error}"- return f"{error} HTTP Error"+ if phrase := HTTP_ERROR_CODES.get(error):+ return f"{error} {phrase}"++ if isinstance(error, int):+ if 300 <= error < 400:+ return f"HTTP Redirection ({error})"+ if 400 <= error < 500:+ return f"HTTP Client Error ({error})"+ if 500 <= error < 600:+ return f"HTTP Server Error ({error})"++ return f"Unknown ({error})"def get_origin(origin: ScrapeItem | Path | MediaItem | URL | None = None) -> Path | URL | None:@@ -235,17 +253,17 @@ def get_origin(origin: ScrapeItem | Path | MediaItem | URL | None = None) -> Patreturn origin[email protected](slots=True)class ErrorLogMessage:ui_failure: str- main_log_msg: str- csv_log_msg: str+ main_log_msg: str = ""+ csv_log_msg: str = ""- def __init__(self, ui_failure: str, main_log_msg: str = "", csv_log_msg: str = "") -> None:- self.ui_failure = ui_failure- self.main_log_msg = main_log_msg or ui_failure- self.csv_log_msg = csv_log_msg or ui_failure+ def __post_init__(self) -> None:+ self.main_log_msg = self.main_log_msg or self.ui_failure+ self.csv_log_msg = self.csv_log_msg or self.ui_failureif self.csv_log_msg == "Unknown":- self.csv_log_msg = "See Logs for details"+ self.csv_log_msg = "See logs for details"@staticmethoddef from_unknown_exc(e: Exception) -> ErrorLogMessage:diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 5454bc8e1..6f934a28a 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -260,22 +260,33 @@ class ClientManager:If the response is successful and has valid html, returns soup"""status: int = response.status_code if hasattr(response, "status_code") else response.status # type: ignore+ content_type: str = getattr(response, "content_type", None) or response.headers.get("Content-Type", "")headers = response.headersurl_host: str = AbsoluteHttpURL(response.url).hostmessage = Nonedef check_etag() -> None:if download and (e_tag := headers.get("ETag")) in DOWNLOAD_ERROR_ETAGS:- message = DOWNLOAD_ERROR_ETAGS.get(e_tag)+ message = DOWNLOAD_ERROR_ETAGS[e_tag]raise DownloadError(HTTPStatus.NOT_FOUND, message=message)async def check_ddos_guard() -> BeautifulSoup | None:+ if "html" not in content_type:+ return++ # TODO: use the response text instead of the raw content to prevent double encoding detection+if soup := await get_soup_no_error(response):if cls.check_ddos_guard(soup) or cls.check_cloudflare(soup):raise DDOSGuardErrorreturn soupasync def check_json_status() -> None:+ if "json" not in content_type:+ return++ # TODO: Define these checks inside their actual crawlers+ # and make them register them on instantationif not any(domain in url_host for domain in ("gofile", "imgur")):returndiff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex 349535fb8..c6f6f4b78 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -93,11 +93,12 @@ def error_handling_wrapper(except NotImplementedError as e:error_log_msg = ErrorLogMessage("NotImplemented")exc_info = e- except TimeoutError:- error_log_msg = ErrorLogMessage("Timeout")+ except TimeoutError as e:+ error_log_msg = ErrorLogMessage("Timeout", repr(e))except ClientConnectorError as e:ui_failure = "Client Connector Error"- log_msg = f"Can't connect to {link}. If you're using a VPN, try turning it off \n {e!s}"+ suffix = "" if (link.host or "").startswith(e.host) else f" from {link}"+ log_msg = f"{e}{suffix}. If you're using a VPN, try turning it off"error_log_msg = ErrorLogMessage(ui_failure, log_msg)except ValidationError as e:exc_info = e