Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: improve error messages (#1270)

NTFSvolume15f0d418d6d2
diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.py
index 04dbc8f26..d2c730cad 100644
--- a/cyberdrop_dl/exceptions.py
+++ b/cyberdrop_dl/exceptions.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import dataclasses
from http import HTTPStatus
from pathlib import Path
from typing import TYPE_CHECKING
@@ -20,7 +21,7 @@ def _format_error(ui_failure: str, message: str) -> str:
# See: https://developers.cloudflare.com/support/troubleshooting/cloudflare-errors/troubleshooting-cloudflare-5xx-errors/
-CLOUDFLARE_ERRORS = {
+CLOUDFLARE_HTTP_ERROR_CODES = {
520: "Unexpected Response",
521: "Web Server Down",
522: "Connection Timeout",
@@ -31,6 +32,19 @@ CLOUDFLARE_ERRORS = {
530: "IP Banned / Restricted",
}
+# https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#Nonstandard_codes
+# Not all of them, just the ones we actually expect to see and do not shadow standard onces
+NON_STANDARD_HTTP_ERROR_CODES = {
+ 419: "Page Expired",
+ 509: "Bandwidth Limit Exceeded",
+}
+
+HTTP_ERROR_CODES = {
+ **NON_STANDARD_HTTP_ERROR_CODES,
+ **CLOUDFLARE_HTTP_ERROR_CODES,
+ **{code.value: code.phrase for code in HTTPStatus},
+}
+
class TooManyCrawlerErrors(Exception): # noqa: N818
"""This exception will be raised after a crawler had too many errors proccesing URLs"""
@@ -219,14 +233,18 @@ class InvalidYamlError(CDLBaseError):
def create_error_msg(error: int | str) -> str:
if isinstance(error, str):
return error
- try:
- msg = HTTPStatus(error).phrase
- return f"{error} {msg}"
- except ValueError:
- cloudflare_error = CLOUDFLARE_ERRORS.get(error)
- if cloudflare_error:
- return f"{error} {cloudflare_error}"
- return f"{error} HTTP Error"
+ if phrase := HTTP_ERROR_CODES.get(error):
+ return f"{error} {phrase}"
+
+ if isinstance(error, int):
+ if 300 <= error < 400:
+ return f"HTTP Redirection ({error})"
+ if 400 <= error < 500:
+ return f"HTTP Client Error ({error})"
+ if 500 <= error < 600:
+ return f"HTTP Server Error ({error})"
+
+ return f"Unknown ({error})"
def get_origin(origin: ScrapeItem | Path | MediaItem | URL | None = None) -> Path | URL | None:
@@ -235,17 +253,17 @@ def get_origin(origin: ScrapeItem | Path | MediaItem | URL | None = None) -> Pat
return origin
[email protected](slots=True)
class ErrorLogMessage:
ui_failure: str
- main_log_msg: str
- csv_log_msg: str
+ main_log_msg: str = ""
+ csv_log_msg: str = ""
- def __init__(self, ui_failure: str, main_log_msg: str = "", csv_log_msg: str = "") -> None:
- self.ui_failure = ui_failure
- self.main_log_msg = main_log_msg or ui_failure
- self.csv_log_msg = csv_log_msg or ui_failure
+ def __post_init__(self) -> None:
+ self.main_log_msg = self.main_log_msg or self.ui_failure
+ self.csv_log_msg = self.csv_log_msg or self.ui_failure
if self.csv_log_msg == "Unknown":
- self.csv_log_msg = "See Logs for details"
+ self.csv_log_msg = "See logs for details"
@staticmethod
def from_unknown_exc(e: Exception) -> ErrorLogMessage:
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 5454bc8e1..6f934a28a 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -260,22 +260,33 @@ class ClientManager:
If the response is successful and has valid html, returns soup
"""
status: int = response.status_code if hasattr(response, "status_code") else response.status # type: ignore
+ content_type: str = getattr(response, "content_type", None) or response.headers.get("Content-Type", "")
headers = response.headers
url_host: str = AbsoluteHttpURL(response.url).host
message = None
def check_etag() -> None:
if download and (e_tag := headers.get("ETag")) in DOWNLOAD_ERROR_ETAGS:
- message = DOWNLOAD_ERROR_ETAGS.get(e_tag)
+ message = DOWNLOAD_ERROR_ETAGS[e_tag]
raise DownloadError(HTTPStatus.NOT_FOUND, message=message)
async def check_ddos_guard() -> BeautifulSoup | None:
+ if "html" not in content_type:
+ return
+
+ # TODO: use the response text instead of the raw content to prevent double encoding detection
+
if soup := await get_soup_no_error(response):
if cls.check_ddos_guard(soup) or cls.check_cloudflare(soup):
raise DDOSGuardError
return soup
async def check_json_status() -> None:
+ if "json" not in content_type:
+ return
+
+ # TODO: Define these checks inside their actual crawlers
+ # and make them register them on instantation
if not any(domain in url_host for domain in ("gofile", "imgur")):
return
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index 349535fb8..c6f6f4b78 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -93,11 +93,12 @@ def error_handling_wrapper(
except NotImplementedError as e:
error_log_msg = ErrorLogMessage("NotImplemented")
exc_info = e
- except TimeoutError:
- error_log_msg = ErrorLogMessage("Timeout")
+ except TimeoutError as e:
+ error_log_msg = ErrorLogMessage("Timeout", repr(e))
except ClientConnectorError as e:
ui_failure = "Client Connector Error"
- log_msg = f"Can't connect to {link}. If you're using a VPN, try turning it off \n {e!s}"
+ suffix = "" if (link.host or "").startswith(e.host) else f" from {link}"
+ log_msg = f"{e}{suffix}. If you're using a VPN, try turning it off"
error_log_msg = ErrorLogMessage(ui_failure, log_msg)
except ValidationError as e:
exc_info = e