Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove client manager (part2) (#1784)

NTFSvolume043fd861ea24
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.py
index 5fdddaea0..027897b67 100644
--- a/cyberdrop_dl/clients/__init__.py
+++ b/cyberdrop_dl/clients/__init__.py
@@ -7,7 +7,7 @@ import logging
import time
import uuid
from pathlib import Path
-from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
+from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self, cast
from multidict import CIMultiDict
@@ -261,10 +261,12 @@ def _write_resp_to_disk(
pass
-class HTTPClientProxy:
+class HTTPClientProxy(Protocol):
DOMAIN: ClassVar[str]
_IMPERSONATE: ClassVar[str | bool | None] = None
- client: HTTPClient # pyright: ignore[reportUninitializedInstanceVariable]
+
+ @property
+ def client(self) -> HTTPClient: ...
@classmethod
def __json_resp_check__(cls, json_resp: Any, resp: AbstractResponse[Any], /) -> None:
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 481f6e661..9dd1208cc 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -34,27 +34,18 @@ logger = logging.getLogger(__name__)
_CONTENT_TYPES_OVERRIDES: dict[str, str] = {"text/vnd.trolltech.linguist": "video/MP2T"}
_SLOW_DOWNLOAD_PERIOD: int = 10 # seconds
_FREE_SPACE_CHECK_PERIOD: int = 5 # Check every 5 chunks
-_NULL_CONTEXT: contextlib.nullcontext[None] = contextlib.nullcontext()
_USE_IMPERSONATION: set[str] = {"vsco", "celebforum"}
class DownloadClient:
- """AIOHTTP operations for downloading."""
+ """Low level class that performs the actual HTTP download operations"""
def __init__(self, manager: Manager, client_manager: ClientManager) -> None:
self.manager = manager
self.client_manager = client_manager
self.download_speed_threshold = self.manager.config.settings.runtime_options.slow_download_speed
- self._server_locks = aio.WeakAsyncLocks[str]()
- self.server_locked_domains: set[str] = set()
self._supports_ranges: bool = True
- def server_limiter(self, domain: str, server: str) -> asyncio.Lock | contextlib.nullcontext[None]:
- if domain not in self.server_locked_domains:
- return _NULL_CONTEXT
-
- return self._server_locks[server]
-
async def _download(self, domain: str, media_item: MediaItem) -> bool:
"""Downloads a file."""
downloaded_filename = await self.manager.database.history.get_downloaded_filename(domain, media_item)
@@ -141,7 +132,7 @@ class DownloadClient:
def _get_resp_reader(
self, resp: aiohttp.ClientResponse | AbstractResponse[Any]
- ) -> AbstractResponse | aiohttp.StreamReader:
+ ) -> AbstractResponse[Any] | aiohttp.StreamReader:
if isinstance(resp, AbstractResponse):
return resp
return resp.content
diff --git a/cyberdrop_dl/crawlers/_one_manager.py b/cyberdrop_dl/crawlers/_one_manager.py
index de1a448c1..8668743c3 100644
--- a/cyberdrop_dl/crawlers/_one_manager.py
+++ b/cyberdrop_dl/crawlers/_one_manager.py
@@ -40,7 +40,7 @@ class OneManagerCrawler(Crawler, is_abc=True):
await self._path(scrape_item)
async def __async_post_init__(self) -> None:
- self.manager.client_manager.download_slots.update({self.DOMAIN: 2})
+ self.downloader.download_slots = 2
@error_handling_wrapper
async def _path(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 63057baa9..fb7a8b575 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -23,7 +23,7 @@ from typing_extensions import deprecated
from cyberdrop_dl import env
from cyberdrop_dl.clients import HTTPClient, HTTPClientProxy
from cyberdrop_dl.crawlers._hls import HLSParser
-from cyberdrop_dl.downloader.downloader import Downloader
+from cyberdrop_dl.downloader.http import Downloader
from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError
from cyberdrop_dl.mediaprops import ISO639Subtitle, Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
@@ -172,17 +172,27 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
@final
def __init__(self, manager: Manager) -> None:
self.manager: Manager = manager
- self.downloader: Downloader = dataclasses.field(init=False)
- self.client: HTTPClient = dataclasses.field(init=False)
+
self._startup_lock: asyncio.Lock = asyncio.Lock()
self._ready: bool = False
self._logged_in: bool = False
self._scraped_items: set[str] = set()
-
self._logger: _CrawlerLogger = _CrawlerLogger(self.FOLDER_DOMAIN)
self._semaphore: asyncio.Semaphore = asyncio.Semaphore(20)
+
+ self.downloader: Downloader = Downloader(
+ self.manager,
+ self.DOMAIN,
+ use_server_lock=self._USE_DOWNLOAD_SERVERS_LOCKS,
+ download_slots=self._DOWNLOAD_SLOTS,
+ )
+
self.__post_init__()
+ @property
+ def client(self) -> HTTPClient:
+ return self.manager.client_manager.scraper_client
+
def __post_init__(self) -> None:
"""Override in subclasses to add custom init logic
@@ -195,13 +205,8 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
async with self._startup_lock:
if self._ready:
return
- self.client = self.manager.client_manager.scraper_client
self.manager.client_manager.rate_limits[self.DOMAIN] = AsyncLimiter(*self._RATE_LIMIT)
- if self._DOWNLOAD_SLOTS:
- self.manager.client_manager.download_slots[self.DOMAIN] = self._DOWNLOAD_SLOTS
- if self._USE_DOWNLOAD_SERVERS_LOCKS:
- self.manager.client_manager.download_client.server_locked_domains.add(self.DOMAIN)
- self.__init_downloader__()
+
await self.__async_post_init__()
self._ready = True
@@ -276,10 +281,6 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
if add_to_register:
Registry.concrete.add(cls)
- def __init_downloader__(self) -> None:
- self.downloader = dl = Downloader(self.manager, self.DOMAIN)
- dl.startup()
-
@final
@staticmethod
def _assert_fields_overrides(subclass: type[Crawler], *fields: str) -> None:
diff --git a/cyberdrop_dl/crawlers/http_direct.py b/cyberdrop_dl/crawlers/http_direct.py
index cfac9a4b9..a5c693455 100644
--- a/cyberdrop_dl/crawlers/http_direct.py
+++ b/cyberdrop_dl/crawlers/http_direct.py
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.constants import FileExt
from cyberdrop_dl.crawlers.crawler import Crawler
+from cyberdrop_dl.downloader.http import Downloader
from cyberdrop_dl.exceptions import NoExtensionError
from cyberdrop_dl.utils.filepath import get_filename_and_ext
@@ -14,6 +15,10 @@ if TYPE_CHECKING:
class DirectHttpFile(Crawler, is_generic=True):
DOMAIN: ClassVar[str] = "no_crawler"
+ async def __async_post_init__(self) -> None:
+ self.downloader = Downloader(self.manager, self.DOMAIN)
+ self.downloader.log_prefix = "Download attempt (unsupported domain)"
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
try:
filename, ext = get_filename_and_ext(scrape_item.url.name)
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index 6f1d5e0cb..2ce75a85f 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -13,7 +13,6 @@ from mega.api import MegaAPI
from mega.core import MegaCore
from mega.crypto import b64_to_a32
from mega.data_structures import Crypto
-from typing_extensions import override
from cyberdrop_dl.constants import CDL_USER_AGENT
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_id
@@ -60,15 +59,11 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
def password(self) -> str | None:
return self.manager.config.auth.meganz.password or None
- @override
- def __init_downloader__(self) -> None:
+ async def __async_post_init__(self) -> None:
api = MegaAPI(self.manager.client_manager._session)
api.user_agent = CDL_USER_AGENT
self.core = MegaCore(api)
- self.downloader = dl = MegaDownloader(self.manager, self.DOMAIN) # type: ignore[reportIncompatibleVariableOverride]
- dl.startup()
-
- async def __async_post_init__(self) -> None:
+ self.downloader = MegaDownloader(self.manager, self.DOMAIN) # pyright: ignore[reportIncompatibleVariableOverride]
await self.login(self.PRIMARY_URL)
async def fetch(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/http.py
similarity index 81%
rename from cyberdrop_dl/downloader/downloader.py
rename to cyberdrop_dl/downloader/http.py
index 0ef72c8d4..4615a0d35 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/http.py
@@ -2,9 +2,9 @@ from __future__ import annotations
import asyncio
import contextlib
+import dataclasses
import logging
import os
-from dataclasses import field
from typing import TYPE_CHECKING, NamedTuple
from aiohttp import ClientConnectorError, ClientError, ClientResponseError
@@ -13,7 +13,6 @@ from cyberdrop_dl import aio, constants, ffmpeg, storage
from cyberdrop_dl.exceptions import (
DownloadError,
DurationError,
- ErrorLogMessage,
InsufficientFreeSpaceError,
InvalidContentTypeError,
RestrictedDateRangeError,
@@ -25,7 +24,7 @@ from cyberdrop_dl.utils import dates, error_handling_wrapper, parse_url
if TYPE_CHECKING:
import datetime
- from collections.abc import Generator
+ from collections.abc import AsyncGenerator, Generator
from pathlib import Path
from cyberdrop_dl.clients.download_client import DownloadClient
@@ -52,22 +51,63 @@ _KNOWN_BAD_URLS = {
_GENERIC_CRAWLERS = ".", "no_crawler"
+_FILE_LOCKS: aio.WeakAsyncLocks[str] = aio.WeakAsyncLocks()
+_NULL_CONTEXT: contextlib.nullcontext[None] = contextlib.nullcontext()
+async def _exclusive_lock(media_item: MediaItem) -> AsyncGenerator[None]:
+ async with _FILE_LOCKS[media_item.filename]:
+ logger.debug(f"Lock for '{media_item.filename}' acquired")
+ try:
+ yield
+ finally:
+ logger.debug(f"Lock for '{media_item.filename}' released")
+
+
[email protected](slots=True)
class Downloader:
- def __init__(self, manager: Manager, domain: str) -> None:
- self.manager: Manager = manager
- self.domain: str = domain
+ """Hight level class that handles limiters, database checks, skip by config checks and retries"""
+
+ manager: Manager
+ domain: str
+ log_prefix: str = "Download"
+ download_slots: int | None = None
+ use_server_lock: bool = False
+
+ waiting_items: int = dataclasses.field(init=False, default=0)
+
+ _processed_items: set[str] = dataclasses.field(init=False, default_factory=set)
+ _current_attempt_filesize: dict[str, int] = dataclasses.field(init=False, default_factory=dict)
+ _semaphore: asyncio.Semaphore = dataclasses.field(init=False)
+ _server_locks: aio.WeakAsyncLocks[str] = dataclasses.field(init=False, default_factory=aio.WeakAsyncLocks)
+
+ def __post_init__(self) -> None:
+ upper_limit = self.config.global_settings.rate_limiting_options.max_simultaneous_downloads_per_domain
+ self._semaphore = asyncio.Semaphore(min(self.download_slots or upper_limit, upper_limit))
- self.client: DownloadClient = field(init=False)
- self._log_prefix = "Download attempt (unsupported domain)" if domain in _GENERIC_CRAWLERS else "Download"
- self._processed_items: set[str] = set()
- self.waiting_items = 0
+ @property
+ def client(self) -> DownloadClient:
+ return self.manager.client_manager.download_client
+
+ @property
+ def config(self) -> Config:
+ return self.manager.config
+
+ @property
+ def _ignore_history(self) -> bool:
+ return self.manager.config.settings.runtime_options.ignore_history
- self._current_attempt_filesize: dict[str, int] = {}
- self._file_locks: aio.WeakAsyncLocks[str] = aio.WeakAsyncLocks()
- self._ignore_history: bool = manager.config.settings.runtime_options.ignore_history
- self._semaphore: asyncio.Semaphore = field(init=False)
+ @property
+ def max_attempts(self) -> int:
+ if self.config.settings.download_options.disable_download_attempt_limit:
+ return 1
+ return self.config.global_settings.rate_limiting_options.download_attempts
+
+ def _server_lock(self, server: str) -> asyncio.Lock | contextlib.nullcontext[None]:
+ if self.use_server_lock:
+ return self._server_locks[server]
+ return _NULL_CONTEXT
@error_handling_wrapper
async def download(self, media_item: MediaItem) -> bool:
@@ -82,29 +122,14 @@ class Downloader:
if e.status != 999:
media_item.attempts += 1
- logger.error(f"{self._log_prefix} failed: {media_item.url} with error: {e!s}")
+ logger.error(f"{self.log_prefix} failed: {media_item.url} with error: {e!s}")
if media_item.attempts >= self.max_attempts:
raise
logger.info(
- f"Retrying {self._log_prefix.lower()}: {media_item.url}, retry attempt: {media_item.attempts + 1}"
+ f"Retrying {self.log_prefix.lower()}: {media_item.url}, retry attempt: {media_item.attempts + 1}"
)
- @property
- def max_attempts(self) -> int:
- if self.manager.config.settings.download_options.disable_download_attempt_limit:
- return 1
- return self.manager.config.global_settings.rate_limiting_options.download_attempts
-
- def startup(self) -> None:
- """Starts the downloader."""
- self.client = self.manager.client_manager.download_client
- self._semaphore = asyncio.Semaphore(self.manager.client_manager.get_download_slots(self.domain))
-
- self.manager.config.settings.files.download_folder.mkdir(parents=True, exist_ok=True)
- if self.manager.config.settings.sorting.sort_downloads:
- self.manager.config.settings.sorting.sort_folder.mkdir(parents=True, exist_ok=True)
-
@contextlib.asynccontextmanager
async def _download_context(self, media_item: MediaItem):
@@ -117,13 +142,11 @@ class Downloader:
self.waiting_items += 1
server = (media_item.debrid_link or media_item.url).host
- server_limit, domain_limit, global_limit = (
- self.client.server_limiter(media_item.domain, server),
+ async with (
+ self._server_lock(server),
self._semaphore,
- self.manager.client_manager.global_download_slots,
- )
-
- async with server_limit, domain_limit, global_limit:
+ self.manager.client_manager.global_download_limiter,
+ ):
self._processed_items.add(media_item.db_path)
self.waiting_items -= 1
yield
@@ -295,11 +318,11 @@ class Downloader:
"""Checks if the file can be downloaded."""
if not await storage.has_sufficient_space(media_item.download_folder):
raise InsufficientFreeSpaceError(media_item)
- if not _is_allowed_filetype(media_item, self.manager.config):
+ if not _is_allowed_filetype(media_item, self.config):
raise RestrictedFiletypeError(origin=media_item)
if not await self.manager.client_manager.check_file_duration(media_item):
raise DurationError(origin=media_item)
- if not _is_allowed_date_range(media_item, self.manager.config):
+ if not _is_allowed_date_range(media_item, self.config):
raise RestrictedDateRangeError(origin=media_item)
async def set_file_datetime(self, media_item: MediaItem, complete_file: Path) -> None:
@@ -307,7 +330,7 @@ class Downloader:
if media_item.is_segment:
return
- if self.manager.config.settings.download_options.disable_file_timestamps:
+ if self.config.settings.download_options.disable_file_timestamps:
return
if not media_item.uploaded_at:
logger.warning(f"Unable to parse upload date for {media_item.url}, using current datetime as file datetime")
@@ -322,18 +345,12 @@ class Downloader:
except OSError:
pass
- """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
-
async def start_download(self, media_item: MediaItem) -> bool:
if not media_item.is_segment:
- logger.info(f"{self._log_prefix} starting: {media_item.url}")
+ logger.info(f"{self.log_prefix} starting: {media_item.url}")
- async with self._file_locks[media_item.filename]:
- logger.debug(f"Lock for '{media_item.filename}' acquired")
- try:
- return bool(await self.download(media_item))
- finally:
- logger.debug(f"Lock for '{media_item.filename}' released")
+ async with _exclusive_lock(media_item):
+ return bool(await self.download(media_item))
async def _download(self, media_item: MediaItem) -> bool | None:
"""Downloads the media item."""
@@ -373,7 +390,7 @@ class Downloader:
ui_message = getattr(e, "status", type(e).__name__)
if media_item.partial_file and (size := await aio.get_size(media_item.partial_file)):
if self._current_attempt_filesize.get(media_item.filename, 0) >= size:
- raise DownloadError(ui_message, message=f"{self._log_prefix} failed", retry=True) from None
+ raise DownloadError(ui_message, message=f"{self.log_prefix} failed", retry=True) from None
self._current_attempt_filesize[media_item.filename] = size
raise DownloadError(status=999, message="Download timeout reached, retrying", retry=True) from None
@@ -381,20 +398,6 @@ class Downloader:
message = str(e)
raise DownloadError(ui_message, message, retry=True) from e
- def write_download_error(
- self,
- media_item: MediaItem,
- error_log_msg: ErrorLogMessage,
- exc_info: Exception | None = None,
- ) -> None:
- logger.error(
- f"{self._log_prefix} Failed: {media_item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {media_item.referer}",
- exc_info=exc_info,
- )
- self.manager.logs.write_download_error(media_item, error_log_msg.csv_log_msg)
- self.manager.scrape_mapper.tui.files.stats.failed += 1
- self.manager.scrape_mapper.tui.download_errors.add(error_log_msg.ui_failure)
-
def _is_allowed_filetype(media_item: MediaItem, config: Config) -> bool:
ignore_options = config.settings.ignore_options
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index bf31c0ad8..66e46020b 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -1,13 +1,14 @@
from __future__ import annotations
-import asyncio
+import dataclasses
from typing import TYPE_CHECKING, final
from mega.chunker import MegaChunker, get_chunks
+from typing_extensions import override
from cyberdrop_dl import aio, storage
from cyberdrop_dl.clients.download_client import DownloadClient
-from cyberdrop_dl.downloader.downloader import Downloader
+from cyberdrop_dl.downloader.http import Downloader
if TYPE_CHECKING:
import aiohttp
@@ -59,17 +60,22 @@ class MegaDownloadClient(DownloadClient):
media_item.partial_file.touch()
[email protected](slots=True)
class MegaDownloader(Downloader):
- client: MegaDownloadClient
+ _client: MegaDownloadClient = dataclasses.field(init=False)
+
+ def __post_init__(self) -> None:
+ super(MegaDownloader, self).__post_init__()
+ self._client = MegaDownloadClient(self.manager)
+
+ @override
+ @property
+ def client(self) -> MegaDownloadClient:
+ return self._client
@property
def max_attempts(self):
return 1
- def startup(self) -> None:
- """Starts the downloader."""
- self.client = MegaDownloadClient(self.manager) # type: ignore[reportIncompatibleVariableOverride]
- self._semaphore = asyncio.Semaphore(self.manager.client_manager.get_download_slots(self.domain))
-
def register(self, url: URL, crypto: Crypto, file_size: int) -> None:
self.client._decrypt_mapping[url] = crypto, file_size
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 4194a6b9a..5be5efe43 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -100,9 +100,8 @@ class ClientManager:
self.ssl_context = _make_ssl_context(self.manager.config.global_settings.general.ssl_context)
self._cookies: aiohttp.CookieJar | None = None
self.rate_limits: dict[str, AsyncLimiter] = {}
- self.download_slots: dict[str, int] = {}
self.global_rate_limiter = AsyncLimiter(self.manager.config.global_settings.rate_limiting_options.rate_limit, 1)
- self.global_download_slots = asyncio.Semaphore(
+ self.global_download_limiter = asyncio.Semaphore(
self.manager.config.global_settings.rate_limiting_options.max_simultaneous_downloads
)
self.scraper_client = HTTPClient.from_client(self)
@@ -170,17 +169,6 @@ class ClientManager:
tg.create_task(close_curl())
- def get_download_slots(self, domain: str) -> int:
- """Returns the download limit for a domain."""
-
- instances = self.download_slots.get(
- domain, self.manager.config.global_settings.rate_limiting_options.max_simultaneous_downloads_per_domain
- )
-
- return min(
- instances, self.manager.config.global_settings.rate_limiting_options.max_simultaneous_downloads_per_domain
- )
-
def _create_curl_session(self) -> AsyncSession[CurlResponse]:
try:
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.py
index 17786732c..777e0930d 100644
--- a/cyberdrop_dl/scrape_mapper.py
+++ b/cyberdrop_dl/scrape_mapper.py
@@ -166,6 +166,10 @@ class ScrapeMapper:
_ = filepath.MAX_FILE_LEN.set(self.manager.config.global_settings.general.max_file_name_length)
_ = filepath.MAX_FOLDER_LEN.set(self.manager.config.global_settings.general.max_folder_name_length)
+ self.manager.config.settings.files.download_folder.mkdir(parents=True, exist_ok=True)
+ if self.manager.config.settings.sorting.sort_downloads:
+ self.manager.config.settings.sorting.sort_folder.mkdir(parents=True, exist_ok=True)
+
await self.manager.client_manager.load_cookie_files()
self.tui.mode = self.manager.cli_args.ui
## IMPORTANT: Order of each context matters!
@@ -194,7 +198,7 @@ class ScrapeMapper:
logger.exception("Failed to connect to jDownloader")
await self._real_debrid.__async_init__()
- self._direct_http.__init_downloader__()
+ await self._direct_http.__async_post_init__()
item_limit = 0
if self.manager.cli_args.retry_any and self.manager.cli_args.max_items_retry:
diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.py
index ee8ecb091..a5eb90bff 100644
--- a/cyberdrop_dl/utils/__init__.py
+++ b/cyberdrop_dl/utils/__init__.py
@@ -34,7 +34,7 @@ from cyberdrop_dl.exceptions import (
if TYPE_CHECKING:
from collections.abc import Callable, Coroutine, Generator
- from cyberdrop_dl.downloader.downloader import Downloader
+ from cyberdrop_dl.downloader.http import Downloader
from cyberdrop_dl.manager import Manager
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
@@ -141,7 +141,13 @@ def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yar
origin = origin or get_origin(item)
if is_downloader:
self, item = cast("Downloader", self), cast("MediaItem", item)
- self.write_download_error(item, error_log_msg, exc_info)
+ logger.error(
+ f"{self.log_prefix} Failed: {item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {item.referer}",
+ exc_info=exc_info,
+ )
+ self.manager.logs.write_download_error(item, error_log_msg.csv_log_msg)
+ self.manager.scrape_mapper.tui.files.stats.failed += 1
+ self.manager.scrape_mapper.tui.download_errors.add(error_log_msg.ui_failure)
return
logger.error(f"Scrape Failed: {link_to_show} ({error_log_msg.main_log_msg})", exc_info=exc_info)