Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove client manager (part2) (#1784)
NTFSvolume043fd861ea24
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.pyindex 5fdddaea0..027897b67 100644--- a/cyberdrop_dl/clients/__init__.py+++ b/cyberdrop_dl/clients/__init__.py@@ -7,7 +7,7 @@ import loggingimport timeimport uuidfrom pathlib import Path-from typing import TYPE_CHECKING, Any, ClassVar, Self, cast+from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self, castfrom multidict import CIMultiDict@@ -261,10 +261,12 @@ def _write_resp_to_disk(pass-class HTTPClientProxy:+class HTTPClientProxy(Protocol):DOMAIN: ClassVar[str]_IMPERSONATE: ClassVar[str | bool | None] = None- client: HTTPClient # pyright: ignore[reportUninitializedInstanceVariable]++ @property+ def client(self) -> HTTPClient: ...@classmethoddef __json_resp_check__(cls, json_resp: Any, resp: AbstractResponse[Any], /) -> None:diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 481f6e661..9dd1208cc 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -34,27 +34,18 @@ logger = logging.getLogger(__name__)_CONTENT_TYPES_OVERRIDES: dict[str, str] = {"text/vnd.trolltech.linguist": "video/MP2T"}_SLOW_DOWNLOAD_PERIOD: int = 10 # seconds_FREE_SPACE_CHECK_PERIOD: int = 5 # Check every 5 chunks-_NULL_CONTEXT: contextlib.nullcontext[None] = contextlib.nullcontext()_USE_IMPERSONATION: set[str] = {"vsco", "celebforum"}class DownloadClient:- """AIOHTTP operations for downloading."""+ """Low level class that performs the actual HTTP download operations"""def __init__(self, manager: Manager, client_manager: ClientManager) -> None:self.manager = managerself.client_manager = client_managerself.download_speed_threshold = self.manager.config.settings.runtime_options.slow_download_speed- self._server_locks = aio.WeakAsyncLocks[str]()- self.server_locked_domains: set[str] = set()self._supports_ranges: bool = True- def server_limiter(self, domain: str, server: str) -> asyncio.Lock | contextlib.nullcontext[None]:- if domain not in self.server_locked_domains:- return _NULL_CONTEXT-- return self._server_locks[server]-async def _download(self, domain: str, media_item: MediaItem) -> bool:"""Downloads a file."""downloaded_filename = await self.manager.database.history.get_downloaded_filename(domain, media_item)@@ -141,7 +132,7 @@ class DownloadClient:def _get_resp_reader(self, resp: aiohttp.ClientResponse | AbstractResponse[Any]- ) -> AbstractResponse | aiohttp.StreamReader:+ ) -> AbstractResponse[Any] | aiohttp.StreamReader:if isinstance(resp, AbstractResponse):return respreturn resp.contentdiff --git a/cyberdrop_dl/crawlers/_one_manager.py b/cyberdrop_dl/crawlers/_one_manager.pyindex de1a448c1..8668743c3 100644--- a/cyberdrop_dl/crawlers/_one_manager.py+++ b/cyberdrop_dl/crawlers/_one_manager.py@@ -40,7 +40,7 @@ class OneManagerCrawler(Crawler, is_abc=True):await self._path(scrape_item)async def __async_post_init__(self) -> None:- self.manager.client_manager.download_slots.update({self.DOMAIN: 2})+ self.downloader.download_slots = 2@error_handling_wrapperasync def _path(self, scrape_item: ScrapeItem) -> None:diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 63057baa9..fb7a8b575 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -23,7 +23,7 @@ from typing_extensions import deprecatedfrom cyberdrop_dl import envfrom cyberdrop_dl.clients import HTTPClient, HTTPClientProxyfrom cyberdrop_dl.crawlers._hls import HLSParser-from cyberdrop_dl.downloader.downloader import Downloader+from cyberdrop_dl.downloader.http import Downloaderfrom cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.mediaprops import ISO639Subtitle, Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem@@ -172,17 +172,27 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):@finaldef __init__(self, manager: Manager) -> None:self.manager: Manager = manager- self.downloader: Downloader = dataclasses.field(init=False)- self.client: HTTPClient = dataclasses.field(init=False)+self._startup_lock: asyncio.Lock = asyncio.Lock()self._ready: bool = Falseself._logged_in: bool = Falseself._scraped_items: set[str] = set()-self._logger: _CrawlerLogger = _CrawlerLogger(self.FOLDER_DOMAIN)self._semaphore: asyncio.Semaphore = asyncio.Semaphore(20)++ self.downloader: Downloader = Downloader(+ self.manager,+ self.DOMAIN,+ use_server_lock=self._USE_DOWNLOAD_SERVERS_LOCKS,+ download_slots=self._DOWNLOAD_SLOTS,+ )+self.__post_init__()+ @property+ def client(self) -> HTTPClient:+ return self.manager.client_manager.scraper_client+def __post_init__(self) -> None:"""Override in subclasses to add custom init logic@@ -195,13 +205,8 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):async with self._startup_lock:if self._ready:return- self.client = self.manager.client_manager.scraper_clientself.manager.client_manager.rate_limits[self.DOMAIN] = AsyncLimiter(*self._RATE_LIMIT)- if self._DOWNLOAD_SLOTS:- self.manager.client_manager.download_slots[self.DOMAIN] = self._DOWNLOAD_SLOTS- if self._USE_DOWNLOAD_SERVERS_LOCKS:- self.manager.client_manager.download_client.server_locked_domains.add(self.DOMAIN)- self.__init_downloader__()+await self.__async_post_init__()self._ready = True@@ -276,10 +281,6 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):if add_to_register:Registry.concrete.add(cls)- def __init_downloader__(self) -> None:- self.downloader = dl = Downloader(self.manager, self.DOMAIN)- dl.startup()-@final@staticmethoddef _assert_fields_overrides(subclass: type[Crawler], *fields: str) -> None:diff --git a/cyberdrop_dl/crawlers/http_direct.py b/cyberdrop_dl/crawlers/http_direct.pyindex cfac9a4b9..a5c693455 100644--- a/cyberdrop_dl/crawlers/http_direct.py+++ b/cyberdrop_dl/crawlers/http_direct.py@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.constants import FileExtfrom cyberdrop_dl.crawlers.crawler import Crawler+from cyberdrop_dl.downloader.http import Downloaderfrom cyberdrop_dl.exceptions import NoExtensionErrorfrom cyberdrop_dl.utils.filepath import get_filename_and_ext@@ -14,6 +15,10 @@ if TYPE_CHECKING:class DirectHttpFile(Crawler, is_generic=True):DOMAIN: ClassVar[str] = "no_crawler"+ async def __async_post_init__(self) -> None:+ self.downloader = Downloader(self.manager, self.DOMAIN)+ self.downloader.log_prefix = "Download attempt (unsupported domain)"+async def fetch(self, scrape_item: ScrapeItem) -> None:try:filename, ext = get_filename_and_ext(scrape_item.url.name)diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex 6f1d5e0cb..2ce75a85f 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -13,7 +13,6 @@ from mega.api import MegaAPIfrom mega.core import MegaCorefrom mega.crypto import b64_to_a32from mega.data_structures import Crypto-from typing_extensions import overridefrom cyberdrop_dl.constants import CDL_USER_AGENTfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_id@@ -60,15 +59,11 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):def password(self) -> str | None:return self.manager.config.auth.meganz.password or None- @override- def __init_downloader__(self) -> None:+ async def __async_post_init__(self) -> None:api = MegaAPI(self.manager.client_manager._session)api.user_agent = CDL_USER_AGENTself.core = MegaCore(api)- self.downloader = dl = MegaDownloader(self.manager, self.DOMAIN) # type: ignore[reportIncompatibleVariableOverride]- dl.startup()-- async def __async_post_init__(self) -> None:+ self.downloader = MegaDownloader(self.manager, self.DOMAIN) # pyright: ignore[reportIncompatibleVariableOverride]await self.login(self.PRIMARY_URL)async def fetch(self, scrape_item: ScrapeItem) -> None:diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/http.pysimilarity index 81%rename from cyberdrop_dl/downloader/downloader.pyrename to cyberdrop_dl/downloader/http.pyindex 0ef72c8d4..4615a0d35 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/http.py@@ -2,9 +2,9 @@ from __future__ import annotationsimport asyncioimport contextlib+import dataclassesimport loggingimport os-from dataclasses import fieldfrom typing import TYPE_CHECKING, NamedTuplefrom aiohttp import ClientConnectorError, ClientError, ClientResponseError@@ -13,7 +13,6 @@ from cyberdrop_dl import aio, constants, ffmpeg, storagefrom cyberdrop_dl.exceptions import (DownloadError,DurationError,- ErrorLogMessage,InsufficientFreeSpaceError,InvalidContentTypeError,RestrictedDateRangeError,@@ -25,7 +24,7 @@ from cyberdrop_dl.utils import dates, error_handling_wrapper, parse_urlif TYPE_CHECKING:import datetime- from collections.abc import Generator+ from collections.abc import AsyncGenerator, Generatorfrom pathlib import Pathfrom cyberdrop_dl.clients.download_client import DownloadClient@@ -52,22 +51,63 @@ _KNOWN_BAD_URLS = {_GENERIC_CRAWLERS = ".", "no_crawler"+_FILE_LOCKS: aio.WeakAsyncLocks[str] = aio.WeakAsyncLocks()+_NULL_CONTEXT: contextlib.nullcontext[None] = contextlib.nullcontext()+async def _exclusive_lock(media_item: MediaItem) -> AsyncGenerator[None]:+ async with _FILE_LOCKS[media_item.filename]:+ logger.debug(f"Lock for '{media_item.filename}' acquired")+ try:+ yield+ finally:+ logger.debug(f"Lock for '{media_item.filename}' released")++[email protected](slots=True)class Downloader:- def __init__(self, manager: Manager, domain: str) -> None:- self.manager: Manager = manager- self.domain: str = domain+ """Hight level class that handles limiters, database checks, skip by config checks and retries"""++ manager: Manager+ domain: str+ log_prefix: str = "Download"+ download_slots: int | None = None+ use_server_lock: bool = False++ waiting_items: int = dataclasses.field(init=False, default=0)++ _processed_items: set[str] = dataclasses.field(init=False, default_factory=set)+ _current_attempt_filesize: dict[str, int] = dataclasses.field(init=False, default_factory=dict)+ _semaphore: asyncio.Semaphore = dataclasses.field(init=False)+ _server_locks: aio.WeakAsyncLocks[str] = dataclasses.field(init=False, default_factory=aio.WeakAsyncLocks)++ def __post_init__(self) -> None:+ upper_limit = self.config.global_settings.rate_limiting_options.max_simultaneous_downloads_per_domain+ self._semaphore = asyncio.Semaphore(min(self.download_slots or upper_limit, upper_limit))- self.client: DownloadClient = field(init=False)- self._log_prefix = "Download attempt (unsupported domain)" if domain in _GENERIC_CRAWLERS else "Download"- self._processed_items: set[str] = set()- self.waiting_items = 0+ @property+ def client(self) -> DownloadClient:+ return self.manager.client_manager.download_client++ @property+ def config(self) -> Config:+ return self.manager.config++ @property+ def _ignore_history(self) -> bool:+ return self.manager.config.settings.runtime_options.ignore_history- self._current_attempt_filesize: dict[str, int] = {}- self._file_locks: aio.WeakAsyncLocks[str] = aio.WeakAsyncLocks()- self._ignore_history: bool = manager.config.settings.runtime_options.ignore_history- self._semaphore: asyncio.Semaphore = field(init=False)+ @property+ def max_attempts(self) -> int:+ if self.config.settings.download_options.disable_download_attempt_limit:+ return 1+ return self.config.global_settings.rate_limiting_options.download_attempts++ def _server_lock(self, server: str) -> asyncio.Lock | contextlib.nullcontext[None]:+ if self.use_server_lock:+ return self._server_locks[server]+ return _NULL_CONTEXT@error_handling_wrapperasync def download(self, media_item: MediaItem) -> bool:@@ -82,29 +122,14 @@ class Downloader:if e.status != 999:media_item.attempts += 1- logger.error(f"{self._log_prefix} failed: {media_item.url} with error: {e!s}")+ logger.error(f"{self.log_prefix} failed: {media_item.url} with error: {e!s}")if media_item.attempts >= self.max_attempts:raiselogger.info(- f"Retrying {self._log_prefix.lower()}: {media_item.url}, retry attempt: {media_item.attempts + 1}"+ f"Retrying {self.log_prefix.lower()}: {media_item.url}, retry attempt: {media_item.attempts + 1}")- @property- def max_attempts(self) -> int:- if self.manager.config.settings.download_options.disable_download_attempt_limit:- return 1- return self.manager.config.global_settings.rate_limiting_options.download_attempts-- def startup(self) -> None:- """Starts the downloader."""- self.client = self.manager.client_manager.download_client- self._semaphore = asyncio.Semaphore(self.manager.client_manager.get_download_slots(self.domain))-- self.manager.config.settings.files.download_folder.mkdir(parents=True, exist_ok=True)- if self.manager.config.settings.sorting.sort_downloads:- self.manager.config.settings.sorting.sort_folder.mkdir(parents=True, exist_ok=True)-@contextlib.asynccontextmanagerasync def _download_context(self, media_item: MediaItem):@@ -117,13 +142,11 @@ class Downloader:self.waiting_items += 1server = (media_item.debrid_link or media_item.url).host- server_limit, domain_limit, global_limit = (- self.client.server_limiter(media_item.domain, server),+ async with (+ self._server_lock(server),self._semaphore,- self.manager.client_manager.global_download_slots,- )-- async with server_limit, domain_limit, global_limit:+ self.manager.client_manager.global_download_limiter,+ ):self._processed_items.add(media_item.db_path)self.waiting_items -= 1yield@@ -295,11 +318,11 @@ class Downloader:"""Checks if the file can be downloaded."""if not await storage.has_sufficient_space(media_item.download_folder):raise InsufficientFreeSpaceError(media_item)- if not _is_allowed_filetype(media_item, self.manager.config):+ if not _is_allowed_filetype(media_item, self.config):raise RestrictedFiletypeError(origin=media_item)if not await self.manager.client_manager.check_file_duration(media_item):raise DurationError(origin=media_item)- if not _is_allowed_date_range(media_item, self.manager.config):+ if not _is_allowed_date_range(media_item, self.config):raise RestrictedDateRangeError(origin=media_item)async def set_file_datetime(self, media_item: MediaItem, complete_file: Path) -> None:@@ -307,7 +330,7 @@ class Downloader:if media_item.is_segment:return- if self.manager.config.settings.download_options.disable_file_timestamps:+ if self.config.settings.download_options.disable_file_timestamps:returnif not media_item.uploaded_at:logger.warning(f"Unable to parse upload date for {media_item.url}, using current datetime as file datetime")@@ -322,18 +345,12 @@ class Downloader:except OSError:pass- """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""-async def start_download(self, media_item: MediaItem) -> bool:if not media_item.is_segment:- logger.info(f"{self._log_prefix} starting: {media_item.url}")+ logger.info(f"{self.log_prefix} starting: {media_item.url}")- async with self._file_locks[media_item.filename]:- logger.debug(f"Lock for '{media_item.filename}' acquired")- try:- return bool(await self.download(media_item))- finally:- logger.debug(f"Lock for '{media_item.filename}' released")+ async with _exclusive_lock(media_item):+ return bool(await self.download(media_item))async def _download(self, media_item: MediaItem) -> bool | None:"""Downloads the media item."""@@ -373,7 +390,7 @@ class Downloader:ui_message = getattr(e, "status", type(e).__name__)if media_item.partial_file and (size := await aio.get_size(media_item.partial_file)):if self._current_attempt_filesize.get(media_item.filename, 0) >= size:- raise DownloadError(ui_message, message=f"{self._log_prefix} failed", retry=True) from None+ raise DownloadError(ui_message, message=f"{self.log_prefix} failed", retry=True) from Noneself._current_attempt_filesize[media_item.filename] = sizeraise DownloadError(status=999, message="Download timeout reached, retrying", retry=True) from None@@ -381,20 +398,6 @@ class Downloader:message = str(e)raise DownloadError(ui_message, message, retry=True) from e- def write_download_error(- self,- media_item: MediaItem,- error_log_msg: ErrorLogMessage,- exc_info: Exception | None = None,- ) -> None:- logger.error(- f"{self._log_prefix} Failed: {media_item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {media_item.referer}",- exc_info=exc_info,- )- self.manager.logs.write_download_error(media_item, error_log_msg.csv_log_msg)- self.manager.scrape_mapper.tui.files.stats.failed += 1- self.manager.scrape_mapper.tui.download_errors.add(error_log_msg.ui_failure)-def _is_allowed_filetype(media_item: MediaItem, config: Config) -> bool:ignore_options = config.settings.ignore_optionsdiff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex bf31c0ad8..66e46020b 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -1,13 +1,14 @@from __future__ import annotations-import asyncio+import dataclassesfrom typing import TYPE_CHECKING, finalfrom mega.chunker import MegaChunker, get_chunks+from typing_extensions import overridefrom cyberdrop_dl import aio, storagefrom cyberdrop_dl.clients.download_client import DownloadClient-from cyberdrop_dl.downloader.downloader import Downloader+from cyberdrop_dl.downloader.http import Downloaderif TYPE_CHECKING:import aiohttp@@ -59,17 +60,22 @@ class MegaDownloadClient(DownloadClient):media_item.partial_file.touch()[email protected](slots=True)class MegaDownloader(Downloader):- client: MegaDownloadClient+ _client: MegaDownloadClient = dataclasses.field(init=False)++ def __post_init__(self) -> None:+ super(MegaDownloader, self).__post_init__()+ self._client = MegaDownloadClient(self.manager)++ @override+ @property+ def client(self) -> MegaDownloadClient:+ return self._client@propertydef max_attempts(self):return 1- def startup(self) -> None:- """Starts the downloader."""- self.client = MegaDownloadClient(self.manager) # type: ignore[reportIncompatibleVariableOverride]- self._semaphore = asyncio.Semaphore(self.manager.client_manager.get_download_slots(self.domain))-def register(self, url: URL, crypto: Crypto, file_size: int) -> None:self.client._decrypt_mapping[url] = crypto, file_sizediff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 4194a6b9a..5be5efe43 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -100,9 +100,8 @@ class ClientManager:self.ssl_context = _make_ssl_context(self.manager.config.global_settings.general.ssl_context)self._cookies: aiohttp.CookieJar | None = Noneself.rate_limits: dict[str, AsyncLimiter] = {}- self.download_slots: dict[str, int] = {}self.global_rate_limiter = AsyncLimiter(self.manager.config.global_settings.rate_limiting_options.rate_limit, 1)- self.global_download_slots = asyncio.Semaphore(+ self.global_download_limiter = asyncio.Semaphore(self.manager.config.global_settings.rate_limiting_options.max_simultaneous_downloads)self.scraper_client = HTTPClient.from_client(self)@@ -170,17 +169,6 @@ class ClientManager:tg.create_task(close_curl())- def get_download_slots(self, domain: str) -> int:- """Returns the download limit for a domain."""-- instances = self.download_slots.get(- domain, self.manager.config.global_settings.rate_limiting_options.max_simultaneous_downloads_per_domain- )-- return min(- instances, self.manager.config.global_settings.rate_limiting_options.max_simultaneous_downloads_per_domain- )-def _create_curl_session(self) -> AsyncSession[CurlResponse]:try:diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex 17786732c..777e0930d 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -166,6 +166,10 @@ class ScrapeMapper:_ = filepath.MAX_FILE_LEN.set(self.manager.config.global_settings.general.max_file_name_length)_ = filepath.MAX_FOLDER_LEN.set(self.manager.config.global_settings.general.max_folder_name_length)+ self.manager.config.settings.files.download_folder.mkdir(parents=True, exist_ok=True)+ if self.manager.config.settings.sorting.sort_downloads:+ self.manager.config.settings.sorting.sort_folder.mkdir(parents=True, exist_ok=True)+await self.manager.client_manager.load_cookie_files()self.tui.mode = self.manager.cli_args.ui## IMPORTANT: Order of each context matters!@@ -194,7 +198,7 @@ class ScrapeMapper:logger.exception("Failed to connect to jDownloader")await self._real_debrid.__async_init__()- self._direct_http.__init_downloader__()+ await self._direct_http.__async_post_init__()item_limit = 0if self.manager.cli_args.retry_any and self.manager.cli_args.max_items_retry:diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.pyindex ee8ecb091..a5eb90bff 100644--- a/cyberdrop_dl/utils/__init__.py+++ b/cyberdrop_dl/utils/__init__.py@@ -34,7 +34,7 @@ from cyberdrop_dl.exceptions import (if TYPE_CHECKING:from collections.abc import Callable, Coroutine, Generator- from cyberdrop_dl.downloader.downloader import Downloader+ from cyberdrop_dl.downloader.http import Downloaderfrom cyberdrop_dl.manager import Managerfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem@@ -141,7 +141,13 @@ def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yarorigin = origin or get_origin(item)if is_downloader:self, item = cast("Downloader", self), cast("MediaItem", item)- self.write_download_error(item, error_log_msg, exc_info)+ logger.error(+ f"{self.log_prefix} Failed: {item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {item.referer}",+ exc_info=exc_info,+ )+ self.manager.logs.write_download_error(item, error_log_msg.csv_log_msg)+ self.manager.scrape_mapper.tui.files.stats.failed += 1+ self.manager.scrape_mapper.tui.download_errors.add(error_log_msg.ui_failure)returnlogger.error(f"Scrape Failed: {link_to_show} ({error_log_msg.main_log_msg})", exc_info=exc_info)