Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: use new scraping UI (#148)
NTFSvolume5e95ffa78beb
diff --git a/cyberdrop_dl/__main__.py b/cyberdrop_dl/__main__.pyindex 7d6de78a2..8bd1ebcad 100644--- a/cyberdrop_dl/__main__.py+++ b/cyberdrop_dl/__main__.py@@ -10,14 +10,13 @@ from typing import Annotatedfrom cyclopts import App, Parameter-from cyberdrop_dl import __version__, aio, webhook+from cyberdrop_dl import __version__, aio, program_ui, webhookfrom cyberdrop_dl.cli import CLIargsfrom cyberdrop_dl.config import Configfrom cyberdrop_dl.logs import log_spacer, setup_console_logging, setup_file_loggingfrom cyberdrop_dl.managers.manager import AppData, Managerfrom cyberdrop_dl.models.types import HttpURLfrom cyberdrop_dl.scrape_mapper import ScrapeMapper-from cyberdrop_dl.ui import program_uifrom cyberdrop_dl.utils import apprise, check_latest_pypifrom cyberdrop_dl.utils.sorting import Sorterfrom cyberdrop_dl.utils.utilities import check_partials_and_empty_folders@@ -33,14 +32,13 @@ async def _scrape(manager: Manager) -> None:async with manager.database:log_spacer()logger.info("Starting CDL...")- with manager.live_manager.get_main_live(stop=True):- async with ScrapeMapper(manager)() as scrape_mapper:- stats = await scrape_mapper.run()+ async with ScrapeMapper(manager)() as scrape_mapper:+ stats = await scrape_mapper.run()log_spacer()await _post_runtime(manager)- stats_summary = manager.progress_manager.print_stats(stats)+ stats_summary = manager.print_stats(stats)log_spacer()await check_latest_pypi()diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex c0eb17c12..ed6dda87a 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -27,6 +27,7 @@ if TYPE_CHECKING:from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItemfrom cyberdrop_dl.managers.client_manager import ClientManagerfrom cyberdrop_dl.managers.manager import Manager+ from cyberdrop_dl.progress import ProgressHooklogger = logging.getLogger(__name__)@@ -102,13 +103,13 @@ class DownloadClient:proceed, skip = await self.get_final_file_info(media_item, domain)self.client_manager.check_content_length(resp.headers)if skip:- self.manager.progress_manager.download_progress.add_skipped()+ self.manager.scrape_mapper.tui.files.stats.skipped += 1return Falseif not proceed:if media_item.is_segment:return Truelogger.info(f"Skipping {media_item.url} as it has already been downloaded")- self.manager.progress_manager.download_progress.add_previously_completed(False)+ self.manager.scrape_mapper.tui.files.stats.previously_completed += 1await self.process_completed(media_item, domain)await self.handle_media_item_completion(media_item, downloaded=False)@@ -127,21 +128,26 @@ class DownloadClient:)media_item.uploaded_at = last_modified- task_id = media_item.task_id- if task_id is None:+ if media_item.is_segment:+ hook = self.manager.scrape_mapper.tui.downloads.download_hls_seg()++ else:size = (media_item.filesize + resume_point) if media_item.filesize is not None else None- task_id = self.manager.progress_manager.file_progress.add_task(- domain=domain, filename=media_item.filename, expected_size=size+ hook = self.manager.scrape_mapper.tui.downloads.download_file(+ media_item.filename,+ media_item.domain,+ size,)- media_item.set_task_id(task_id)- self.manager.progress_manager.file_progress.advance_file(task_id, resume_point)+ if resume_point:+ hook.advance(resume_point)- await self._append_content(media_item, self._get_resp_reader(resp))+ with hook:+ await self._append_content(media_item, hook, self._get_resp_reader(resp))return Truedef _get_resp_reader(- self, resp: aiohttp.ClientResponse | AbstractResponse+ self, resp: aiohttp.ClientResponse | AbstractResponse[Any]) -> AbstractResponse | aiohttp.StreamReader:if isinstance(resp, AbstractResponse):return resp@@ -165,7 +171,7 @@ class DownloadClient:async def _request_download(self,media_item: MediaItem,- process_response: Callable[[aiohttp.ClientResponse | AbstractResponse], Coroutine[None, None, bool]],+ process_response: Callable[[aiohttp.ClientResponse | AbstractResponse[Any]], Coroutine[None, None, bool]],) -> bool:download_url = media_item.debrid_link or media_item.url@@ -200,13 +206,15 @@ class DownloadClient:raiseasync def _append_content(- self, media_item: MediaItem, content: aiohttp.StreamReader | AbstractResponse[Any]+ self,+ media_item: MediaItem,+ hook: ProgressHook,+ content: aiohttp.StreamReader | AbstractResponse[Any],) -> None:"""Appends content to a file."""- assert media_item.task_id is not Nonecheck_free_space = storage.create_free_space_checker(media_item)- check_download_speed = self.make_speed_checker(media_item)+ check_download_speed = self.make_speed_checker(media_item, hook)await check_free_space()await self._pre_download_check(media_item)@@ -216,7 +224,7 @@ class DownloadClient:chunk_size = len(chunk)await self.client_manager.speed_limiter.acquire(chunk_size)await f.write(chunk)- self.manager.progress_manager.file_progress.advance_file(media_item.task_id, chunk_size)+ hook.advance(chunk_size)check_download_speed()await self._post_download_check(media_item)@@ -232,15 +240,15 @@ class DownloadClient:await aio.unlink(media_item.partial_file, missing_ok=True)raise DownloadError(HTTPStatus.INTERNAL_SERVER_ERROR, message="File is empty")- def make_speed_checker(self, media_item: MediaItem) -> Callable[[], None]:+ def make_speed_checker(self, media_item: MediaItem, hook: ProgressHook) -> Callable[[], None]:last_slow_speed_read = Nonedef check_download_speed() -> None:nonlocal last_slow_speed_readif not self.download_speed_threshold:return- assert media_item.task_id is not None- speed = self.manager.progress_manager.file_progress.get_speed(media_item.task_id)++ speed = hook.get_speed()if speed > self.download_speed_threshold:last_slow_speed_read = Noneelif not last_slow_speed_read:@@ -254,7 +262,7 @@ class DownloadClient:"""Starts a file."""if self.manager.config.settings.download_options.skip_download_mark_completed and not media_item.is_segment:logger.info(f"Download removed {media_item.url} due to mark completed option")- self.manager.progress_manager.download_progress.add_skipped()+ self.manager.scrape_mapper.tui.files.stats.skipped += 1# set completed pathawait self.process_completed(media_item, domain)return False@@ -271,7 +279,7 @@ class DownloadClient:logger.info(f"Download skipped {media_item.url} due to runtime restrictions")await aio.unlink(media_item.path)await self.mark_incomplete(media_item, domain)- self.manager.progress_manager.download_progress.add_skipped()+ self.manager.scrape_mapper.tui.files.stats.skipped += 1return Falseawait self.process_completed(media_item, domain)await self.handle_media_item_completion(media_item, downloaded=True)diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pyindex b986644f7..6c6bb8f24 100644--- a/cyberdrop_dl/clients/flaresolverr.py+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -14,6 +14,7 @@ from multidict import CIMultiDict, CIMultiDictProxyfrom cyberdrop_dl.data_structures import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import DDOSGuardError+from cyberdrop_dl.progress.scraping import show_msgif TYPE_CHECKING:from collections.abc import Callable, Iterable, Mapping@@ -148,9 +149,11 @@ class FlareSolverrClient:params["postData"] = aiohttp.FormData(data)().decode()async with self._request_lock:- logger.debug(f"Making FlareSolverr request #{self._request_id()} with {params = }")- async with self._aiohttp_session.post(self.url, json=params, **timeout) as response:- return Response.from_dict(await response.json())+ request_id = self._request_id()+ with show_msg(f"Waiting for flaresolver [{request_id}]"):+ logger.debug(f"Making FlareSolverr request #{request_id} with {params = }")+ async with self._aiohttp_session.post(self.url, json=params, **timeout) as response:+ return Response.from_dict(await response.json())async def _create_session(self) -> None:session_id = "cyberdrop-dl"diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 799dcbf5b..574798c7a 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -45,7 +45,6 @@ if TYPE_CHECKING:import yarlfrom bs4 import BeautifulSoup, Tagfrom curl_cffi.requests.impersonate import BrowserTypeLiteral- from rich.progress import TaskIDfrom cyberdrop_dl.managers.manager import Manager@@ -366,15 +365,10 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):raise exc@final- @contextlib.contextmanager- def new_task_id(self, url: AbsoluteHttpURL) -> Generator[TaskID]:+ def new_task_id(self, url: AbsoluteHttpURL):"""Creates a new task_id (shows the URL in the UI and logs)"""self.log.info(f"Scraping {url}")- task_id = self.manager.progress_manager.scraping_progress.add_task(url)- try:- yield task_id- finally:- self.manager.progress_manager.scraping_progress.remove_task(task_id)+ return self.manager.scrape_mapper.tui.scrape.new(url)@final@staticmethod@@ -482,7 +476,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):current_referer, downloaded = await self.manager.database.history.check_complete(self.DOMAIN, db_path)if downloaded:logger.info("Skipping %s as it has already been downloaded", url)- self.manager.progress_manager.download_progress.add_previously_completed()+ self.manager.scrape_mapper.tui.files.stats.previously_completed += 1if referer and url != referer and str(referer) != current_referer:# Update the referer if it has changed so that check_complete_by_referer can work@@ -498,7 +492,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):returnif await self.check_skip_by_config(media_item):- self.manager.progress_manager.download_progress.add_skipped()+ self.manager.scrape_mapper.tui.files.stats.skipped += 1returnself.manager.scrape_mapper.create_download_task(self._download(media_item, m3u8))@@ -535,7 +529,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):downloaded = await self.manager.database.history.check_complete_by_referer(domain, url)if downloaded:logger.info(f"Skipping {url} as it has already been downloaded")- self.manager.progress_manager.download_progress.add_previously_completed()+ self.manager.scrape_mapper.tui.files.stats.previously_completed += 1return downloaded@final@@ -547,7 +541,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):if downloaded:url = _url(scrape_item)logger.info(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded")- self.manager.progress_manager.download_progress.add_previously_completed()+ self.manager.scrape_mapper.tui.files.stats.previously_completed += 1return downloaded@final@@ -595,7 +589,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):url_path = self.__db_path__(url)if album_results.get(url_path) is True:logger.info(f"Skipping {url} as it has already been downloaded")- self.manager.progress_manager.download_progress.add_previously_completed()+ self.manager.scrape_mapper.tui.files.stats.previously_completed += 1return Truereturn Falsediff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex 8084bfe1b..9bd9bb35a 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -19,7 +19,6 @@ if TYPE_CHECKING:from collections.abc import Callable, Generatorimport aiohttp- from rich.progress import TaskIDfrom cyberdrop_dl import signaturefrom cyberdrop_dl.managers.manager import Manager@@ -158,8 +157,6 @@ class MediaItem:hash: str | None = Nonedownloaded: bool = field(default=False)- parent_media_item: MediaItem | None = None- _task_id: TaskID | None = Nonemetadata: object = field(init=False, default_factory=dict)uploaded_at_date: datetime.datetime | None = field(init=False, default=None)@@ -217,27 +214,9 @@ class MediaItem:original_filename=original_filename or filename,parents=origin.parents.copy(),uploaded_at=origin.uploaded_at,- parent_media_item=None if isinstance(origin, ScrapeItem) else origin,parent_threads=origin.parent_threads.copy(),)- @property- def task_id(self) -> TaskID | None:- if self.parent_media_item is not None:- return self.parent_media_item.task_id- return self._task_id-- def set_task_id(self, task_id: TaskID | None) -> None:- if self.task_id is not None and task_id is not None:- # We already have a task_id; we can't replace it, only reset it.- # This should never happen. Calling code should always check the value before making a new task.- # We can't silently ignore it either because we will lose any reference to the created task.- raise ValueError("task_id is already set")- if self.parent_media_item is not None:- self.parent_media_item.set_task_id(task_id)- else:- self._task_id = task_id-def serialize(self) -> dict[str, Any]:me = asdict(self)if self.hash:diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex e554c7208..efe9d3ebf 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -81,7 +81,6 @@ class Downloader:if not e.retry:raise- self.attempt_task_removal(media_item)if e.status != 999:media_item.attempts += 1@@ -108,11 +107,6 @@ class Downloader:if self.manager.config.settings.sorting.sort_downloads:self.manager.config.settings.sorting.sort_folder.mkdir(parents=True, exist_ok=True)- def update_queued_files(self, increase_total: bool = True):- queued_files = self.manager.progress_manager.file_progress.get_queue_length()- self.manager.progress_manager.download_progress.update_queued(queued_files)- self.manager.progress_manager.download_progress.update_total(increase_total)-@contextlib.asynccontextmanagerasync def _download_context(self, media_item: MediaItem):@@ -123,7 +117,6 @@ class Downloader:returnself.waiting_items += 1- self.update_queued_files()server = (media_item.debrid_link or media_item.url).hostserver_limit, domain_limit, global_limit = (@@ -134,7 +127,6 @@ class Downloader:async with server_limit, domain_limit, global_limit:self.processed_items.add(media_item.db_path)- self.update_queued_files(increase_total=False)self.waiting_items -= 1yield@@ -167,23 +159,27 @@ class Downloader:# TODO: compute approx size for UI from the m3u8 infomedia_item.download_filename = media_item.path.nameawait self.manager.database.history.add_download_filename(self.domain, media_item)- task_id = self.manager.progress_manager.file_progress.add_task(domain=self.domain, filename=media_item.filename)- media_item.set_task_id(task_id)- video, audio, _subs = await self._download_rendition_group(media_item, m3u8_group)- if not audio:- await aio.move(video, media_item.path)- else:- # TODO: add remux method to ffmpeg to create an mkv file instead of mp4- # Subtitles format may be incompatible with mp4 and they will be silently dropped by ffmpeg- # so we leave them as independent files for now- ffmpeg_result = await ffmpeg.merge((video, audio), media_item.path)-- if not ffmpeg_result.success:- raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)-- await self.client.process_completed(media_item, self.domain)- await self.client.handle_media_item_completion(media_item, downloaded=True)- await self.finalize_download(media_item, downloaded=True)++ with self.manager.scrape_mapper.tui.downloads.download_hls(+ media_item.filename,+ media_item.domain,+ segments=sum(len(m.segments) for m in m3u8_group if m is not None),+ ):+ video, audio, _subs = await self._download_rendition_group(media_item, m3u8_group)+ if not audio:+ await aio.move(video, media_item.path)+ else:+ # TODO: add remux method to ffmpeg to create an mkv file instead of mp4+ # Subtitles format may be incompatible with mp4 and they will be silently dropped by ffmpeg+ # so we leave them as independent files for now+ ffmpeg_result = await ffmpeg.merge((video, audio), media_item.path)++ if not ffmpeg_result.success:+ raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)++ await self.client.process_completed(media_item, self.domain)+ await self.client.handle_media_item_completion(media_item, downloaded=True)+ await self.finalize_download(media_item, downloaded=True)async def _download_rendition_group(self, media_item: MediaItem, m3u8_group: Rendition@@ -283,8 +279,7 @@ class Downloader:if downloaded:await aio.chmod(media_item.path, 0o666)await self.set_file_datetime(media_item, media_item.path)- self.attempt_task_removal(media_item)- self.manager.progress_manager.download_progress.add_completed()+ self.manager.scrape_mapper.tui.files.stats.completed += 1logger.info(f"Download finished: {media_item.url}")"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""@@ -320,18 +315,6 @@ class Downloader:except OSError:pass- def attempt_task_removal(self, media_item: MediaItem) -> None:- """Attempts to remove the task from the progress bar."""- if media_item.is_segment:- return- if media_item.task_id is not None:- try:- self.manager.progress_manager.file_progress.remove_task(media_item.task_id)- except ValueError:- pass-- media_item.set_task_id(None)-"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""async def start_download(self, media_item: MediaItem) -> bool:@@ -366,16 +349,14 @@ class Downloader:await aio.chmod(media_item.path, 0o666)if not media_item.is_segment:await self.set_file_datetime(media_item, media_item.path)- self.manager.progress_manager.download_progress.add_completed()+ self.manager.scrape_mapper.tui.files.stats.completed += 1logger.info(f"Download finished: {media_item.url}")- self.attempt_task_removal(media_item)return downloadedexcept SkipDownloadError as e:if not media_item.is_segment:logger.info(f"Download skipped {media_item.url}: {e}")- self.manager.progress_manager.download_progress.add_skipped()- self.attempt_task_removal(media_item)+ self.manager.scrape_mapper.tui.files.stats.skipped += 1except (DownloadError, ClientResponseError, InvalidContentTypeError):raise@@ -405,11 +386,10 @@ class Downloader:error_log_msg: ErrorLogMessage,exc_info: Exception | None = None,) -> None:- self.attempt_task_removal(media_item)logger.error(f"{self.log_prefix} Failed: {media_item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {media_item.referer}",exc_info=exc_info,)self.manager.logs.write_download_error(media_item, error_log_msg.csv_log_msg)- self.manager.progress_manager.download_stats_progress.add_failure(error_log_msg.ui_failure)- self.manager.progress_manager.download_progress.add_failed()+ self.manager.scrape_mapper.tui.files.stats.failed += 1+ self.manager.scrape_mapper.tui.scrape_errors.add(error_log_msg.ui_failure)diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex 39d4bcae4..b552ca4ab 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -1,7 +1,7 @@from __future__ import annotationsimport asyncio-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, finalimport aiofilesfrom mega.chunker import MegaChunker, get_chunks@@ -17,20 +17,21 @@ if TYPE_CHECKING:from cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.manager import Manager+ from cyberdrop_dl.progress import ProgressHook+@finalclass MegaDownloadClient(DownloadClient):def __init__(self, manager: Manager) -> None:super().__init__(manager, manager.client_manager)self._decrypt_mapping: dict[URL, tuple[Crypto, int]] = {}self._supports_ranges = False- async def _append_content(self, media_item: MediaItem, content: aiohttp.StreamReader) -> None:+ async def _append_content(self, media_item: MediaItem, hook: ProgressHook, content: aiohttp.StreamReader) -> None:"""Appends content to a file."""- assert media_item.task_id is not Nonecheck_free_space = storage.create_free_space_checker(media_item)- check_download_speed = self.make_speed_checker(media_item)+ check_download_speed = self.make_speed_checker(media_item, hook)await check_free_space()await self._pre_download_check(media_item)@@ -46,7 +47,7 @@ class MegaDownloadClient(DownloadClient):await self.client_manager.speed_limiter.acquire(chunk_size)await f.write(chunk)- self.manager.progress_manager.file_progress.advance_file(media_item.task_id, chunk_size)+ hook.advance(chunk_size)check_download_speed()await self._post_download_check(media_item)diff --git a/cyberdrop_dl/hasher.py b/cyberdrop_dl/hasher.pyindex c8ca560b2..673bcad31 100644--- a/cyberdrop_dl/hasher.py+++ b/cyberdrop_dl/hasher.py@@ -51,7 +51,7 @@ async def hash_directory_scanner(manager: Manager, path: Path) -> None:async with manager.database:stats = await manager.hasher.hash_directory(path)- manager.progress_manager.print_hashing_stats(stats)+ manager.print_hashing_stats(stats)@dataclasses.dataclass(slots=True)@@ -68,7 +68,7 @@ class Hasher:_dedupe_tui: DedupeUI = dataclasses.field(init=False)def __post_init__(self) -> None:- base_dir = self.manager.config.settings.files.download_folder+ base_dir = self.manager.config.settings.files.download_folder.expanduser().resolve().absolute()self._tui = HashingUI(base_dir)self._dedupe_tui = DedupeUI(base_dir)diff --git a/cyberdrop_dl/managers/live_manager.py b/cyberdrop_dl/managers/live_manager.pydeleted file mode 100644index a0fa7a25c..000000000--- a/cyberdrop_dl/managers/live_manager.py+++ /dev/null@@ -1,104 +0,0 @@-from __future__ import annotations--import asyncio-import shutil-from contextlib import contextmanager-from functools import partialmethod-from typing import TYPE_CHECKING--from rich.live import Live--from cyberdrop_dl import env-from cyberdrop_dl.logs import LOG_TO_CONSOLE--if TYPE_CHECKING:- from collections.abc import Generator-- from rich.console import RenderableType-- from cyberdrop_dl.managers.manager import Manager---def is_terminal_in_portrait() -> bool:- """Check if CDL is being run in portrait mode based on a few conditions."""-- if env.PORTRAIT_MODE:- return True-- terminal_size = shutil.get_terminal_size()- width, height = terminal_size.columns, terminal_size.lines- aspect_ratio = width / height-- # High aspect ratios are likely to be in landscape mode- if aspect_ratio >= 3.2:- return False-- # Check for mobile device in portrait mode- if (aspect_ratio < 1.5 and height >= 40) or (width <= 85 and aspect_ratio < 2.3):- return True-- # Assume landscape mode for other cases- return False---class LiveManager:- def __init__(self, manager: Manager) -> None:- self.manager = manager- self.ui_setting = self.manager.cli_args.ui- self.fullscreen = f = self.manager.cli_args.fullscreen_ui- self.refresh_rate = rate = self.manager.config.global_settings.ui_options.refresh_rate- self.live = Live(refresh_per_second=rate, transient=True, screen=f, auto_refresh=True)- self.current_layout: str = ""-- @contextmanager- def get_live(self, name: str, stop: bool = False) -> Generator[Live | None]:- layout = self.get_layout(name)- token = LOG_TO_CONSOLE.set(False)- try:- with self.live_context_manager(layout, stop=stop) as live:- yield live- finally:- LOG_TO_CONSOLE.reset(token)-- get_sort_live = partialmethod(get_live, name="sort_layout")- get_main_live = partialmethod(get_live, name="main_layout")- get_hash_live = partialmethod(get_live, name="hash_layout")- get_remove_file_via_hash_live = partialmethod(get_live, name="hash_remove_layout")-- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #-- def get_layout(self, name: str) -> RenderableType | None:- if name == "main_layout":- name = f"{self.ui_setting.value}_layout"- self.current_layout = name- return getattr(self.manager.progress_manager, name, None)-- async def watch_orientation(self, stop_event: asyncio.Event) -> None:- """Watches for screen orientation changes and updates the live display accordingly."""- while not stop_event.is_set():- new_layout = "vertical_layout" if is_terminal_in_portrait() else "horizontal_layout"- if new_layout != self.current_layout:- self.current_layout = new_layout- layout = self.get_layout(new_layout)- self.live.update(layout, refresh=True) # type: ignore[reportArgumentType]- await asyncio.sleep(0.5)-- @contextmanager- def live_context_manager(self, layout: RenderableType | None, stop: bool = False) -> Generator[Live | None]:- stop_event = asyncio.Event()- orientation_task = None-- try:- self.live.start()- if layout:- self.live.update(layout, refresh=True)- if self.current_layout in ("vertical_layout", "horizontal_layout"):- orientation_task = asyncio.create_task(self.watch_orientation(stop_event))- yield self.live- finally:- stop_event.set()- if orientation_task:- orientation_task.cancel()- if stop:- self.live.update("")- self.live.stop()diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex a3378c798..81486bb1f 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -2,25 +2,33 @@ from __future__ import annotationsimport dataclassesimport logging+import time+from datetime import timedeltafrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Self+from pydantic.types import ByteSize+from cyberdrop_dl import __version__, ffmpeg, yamlfrom cyberdrop_dl.cli import CLIargsfrom cyberdrop_dl.config import Configfrom cyberdrop_dl.database import Databasefrom cyberdrop_dl.hasher import Hasher+from cyberdrop_dl.logs import capture_logs, log_spacerfrom cyberdrop_dl.managers.client_manager import ClientManager-from cyberdrop_dl.managers.live_manager import LiveManagerfrom cyberdrop_dl.managers.logs import LogManager-from cyberdrop_dl.managers.progress_manager import ProgressManagerfrom cyberdrop_dl.utils.utilities import get_system_informationif TYPE_CHECKING:+ from collections.abc import Sequencefrom os import PathLikefrom cyberdrop_dl.data_structures.url_objects import MediaItem- from cyberdrop_dl.scrape_mapper import ScrapeMapper+ from cyberdrop_dl.progress.dedupe import DedupeStats+ from cyberdrop_dl.progress.hashing import HashingStats+ from cyberdrop_dl.progress.scraping.errors import Error+ from cyberdrop_dl.progress.sorting import SortStats+ from cyberdrop_dl.scrape_mapper import ScrapeMapper, ScrapeStatslogger = logging.getLogger(__name__)@@ -42,9 +50,6 @@ class Manager:self._completed_downloads: list[MediaItem] = []self.hasher: Hasher = Hasher(self)self.logs: LogManager = LogManager.from_manager(self)- self.live_manager: LiveManager = LiveManager(self)- self.progress_manager: ProgressManager = ProgressManager(self)-self.scrape_mapper: ScrapeMapperself.database: Databaseself.client_manager: ClientManager@@ -115,6 +120,107 @@ class Manager:async def close(self) -> None:await self.client_manager.close()+ def print_stats(self, stats: ScrapeStats) -> str:+ if not self.cli_args.print_stats:+ return ""++ log_spacer()++ with capture_logs() as stream:+ self._print_stats(stats)++ return stream.getvalue()++ def _print_stats(self, stats: ScrapeStats) -> None:++ elapsed = timedelta(seconds=int(time.monotonic() - stats.start_time))+ total_data_written = ByteSize(self.scrape_mapper.tui.downloads.bytes_downloaded).human_readable(decimal=True)++ logger.info("Run Stats:", extra={"color": "cyan"})+ logger.info(f" Config file: {self.config.source}")+ logger.info(f" URLs source: {stats.source}")+ logger.info(f" URLs: {stats.count:,}")+ logger.info(f" URL groups: {len(stats.unique_groups):,}")+ logger.info(f" Logs folder: {self.config.settings.logs.log_folder}")+ logger.info(f" Total runtime: {elapsed}")+ logger.info(f" Total downloaded data: {total_data_written}")++ if stats.domain_stats:+ log_spacer()+ logger.info("URLs by domain:", extra={"color": "cyan"})+ logger.info(" - " + "\n - ".join(map(str, stats.domain_stats.items())))++ log_spacer()+ logger.info("Download Stats:", extra={"color": "cyan"})+ logger.info(f" Downloaded: {self.scrape_mapper.tui.files.stats.completed:,} files")+ logger.info(f" Skipped (by config): {self.scrape_mapper.tui.files.stats.skipped:,} files")+ logger.info(+ f" Skipped (previously downloaded): {self.scrape_mapper.tui.files.stats.previously_completed:,} files"+ )+ logger.info(f" Failed: {self.scrape_mapper.tui.files.stats.failed:,} files")++ log_spacer()+ logger.info("Unsupported URLs Stats:", extra={"color": "cyan"})+ logger.info(f" Sent to Jdownloader: {self.scrape_mapper.tui.scrape_errors.sent_to_jdownloader:,}")+ logger.info(f" Skipped: {self.scrape_mapper.tui.scrape_errors.skipped:,}")++ hash_stats, dedupe_stats = self.hasher.stats+ self.print_hashing_stats(hash_stats)+ self.print_dedupe_stats(dedupe_stats)+ # self.print_sort_stats()+ self.print_errors()++ def print_sort_stats(self, stats: SortStats):+ log_spacer()+ logger.info("Sort Stats:", extra={"color": "cyan"})+ logger.info(f" Audios: {stats.audios:,}")+ logger.info(f" Images: {stats.images:,}")+ logger.info(f" Videos: {stats.videos:,}")+ logger.info(f" Other Files: {stats.others:,}")++ def print_errors(self):+ _log_errors(+ tuple(self.scrape_mapper.tui.scrape_errors),+ tuple(self.scrape_mapper.tui.download_errors),+ )++ def print_hashing_stats(self, stats: HashingStats) -> None:+ log_spacer()+ logger.info("Checksum Stats:", extra={"color": "cyan"})+ logger.info(f" Newly hashed: {stats.new_hashed:,} files")+ logger.info(f" Previously hashed: {stats.prev_hashed:,} files")++ def print_dedupe_stats(self, stats: DedupeStats) -> None:+ log_spacer()+ logger.info("Dupe Stats:", extra={"color": "cyan"})+ logger.info(f" Deleted (duplicates of previous downloads): {stats.deleted:,} files")+ logger.info(f" Errors: {stats.total - stats.deleted:,} files")+++def _log_errors(scrape_errors: Sequence[Error], download_errors: Sequence[Error]) -> None:+ error_codes = (error.code for error in (*scrape_errors, *download_errors) if error.code is not None)++ try:+ padding = len(str(max(error_codes)))+ except ValueError:+ padding = 0++ for title, errors in (+ ("Scrape Failures:", scrape_errors),+ ("Download Failures:", download_errors),+ ):+ log_spacer()+ logger.info(title, extra={"color": "cyan"})+ if not errors:+ logger.info(f" {'None':>{padding}}", extra={"color": "green"})+ continue++ for error in errors:+ error_code = error.code if error.code is not None else ""+ logger.info(+ f" {error_code:>{padding}}{' ' if padding else ''}{error.msg}: {error.count:,}", extra={"color": "red"}+ )+@dataclasses.dataclass(slots=True, frozen=True, kw_only=True)class AppData:diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.pyindex afe492228..19b253623 100644--- a/cyberdrop_dl/managers/progress_manager.py+++ b/cyberdrop_dl/managers/progress_manager.py@@ -1,211 +1,10 @@from __future__ import annotationsimport logging-import time-from contextlib import asynccontextmanager-from datetime import timedelta-from typing import TYPE_CHECKING--from pydantic import ByteSize-from rich.columns import Columns-from rich.console import Group-from rich.layout import Layout-from rich.progress import Progress, SpinnerColumn, TaskID--from cyberdrop_dl import __version__-from cyberdrop_dl.logs import capture_logs, log_spacer-from cyberdrop_dl.ui.progress.downloads_progress import DownloadsProgress-from cyberdrop_dl.ui.progress.file_progress import FileProgress-from cyberdrop_dl.ui.progress.hash_progress import HashProgress-from cyberdrop_dl.ui.progress.scraping_progress import ScrapingProgress-from cyberdrop_dl.ui.progress.sort_progress import SortProgress-from cyberdrop_dl.ui.progress.statistic_progress import DownloadStatsProgress, ScrapeStatsProgress--if TYPE_CHECKING:- from collections.abc import AsyncGenerator, Sequence-- from rich.panel import Panel-- from cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.progress.dedupe import DedupeStats- from cyberdrop_dl.progress.hashing import HashingStats- from cyberdrop_dl.scrape_mapper import ScrapeStats- from cyberdrop_dl.ui.progress.statistic_progress import UiFailureTotal+from rich.progress import SpinnerColumnlogger = logging.getLogger(__name__)spinner = SpinnerColumn(style="green", spinner_name="dots")---class ProgressManager:- def __init__(self, manager: Manager) -> None:- # File Download Bars- self.manager = manager- ui_options = manager.config.global_settings.ui_options- self.portrait = manager.cli_args.portrait- self.file_progress = FileProgress(manager)- self.scraping_progress = ScrapingProgress(manager)-- # Overall Progress Bars & Stats- self.download_progress: DownloadsProgress = DownloadsProgress(manager)- self.download_stats_progress: DownloadStatsProgress = DownloadStatsProgress()- self.scrape_stats_progress: ScrapeStatsProgress = ScrapeStatsProgress()- self.hash_progress: HashProgress = HashProgress(manager)- self.sort_progress: SortProgress = SortProgress(1, manager)-- self.ui_refresh_rate: int = ui_options.refresh_rate-- activity = Progress(spinner, "[progress.description]{task.description}")- self.status_message: Progress = Progress(spinner, "[progress.description]{task.description}")-- self.status_message_task_id: TaskID = self.status_message.add_task("", total=100, completed=0, visible=False)- self.activity_task_id: TaskID = activity.add_task(- f"Running Cyberdrop-DL: v{__version__}", total=100, completed=0- )- self.activity: Progress = activity-- simple_layout = Group(activity, self.download_progress.simple_progress)-- status_message_columns = Columns([activity, self.status_message], expand=False)-- horizontal_layout = Layout()- vertical_layout = Layout()-- upper_layouts = (- Layout(renderable=self.download_progress.get_progress(), name="Files", ratio=1, minimum_size=9),- Layout(renderable=self.scrape_stats_progress.get_progress(), name="Scrape Failures", ratio=1),- Layout(renderable=self.download_stats_progress.get_progress(), name="Download Failures", ratio=1),- )-- lower_layouts = (- Layout(renderable=self.scraping_progress.get_renderable(), name="Scraping", ratio=20),- Layout(renderable=self.file_progress.get_renderable(), name="Downloads", ratio=20),- Layout(renderable=status_message_columns, name="status_message", ratio=2),- )-- horizontal_layout.split_column(Layout(name="upper", ratio=20), *lower_layouts)- vertical_layout.split_column(Layout(name="upper", ratio=60), *lower_layouts)-- horizontal_layout["upper"].split_row(*upper_layouts)- vertical_layout["upper"].split_column(*upper_layouts)-- self.horizontal_layout: Layout = horizontal_layout- self.vertical_layout: Layout = vertical_layout- self.activity_layout: Progress = activity- self.simple_layout: Group = simple_layout- self.hash_remove_layout: Panel = self.hash_progress.get_removed_progress()- self.hash_layout: Panel = self.hash_progress.get_renderable()- self.sort_layout: Panel = self.sort_progress.get_renderable()-- @asynccontextmanager- async def show_status_msg(self, msg: str | None) -> AsyncGenerator[None]:- try:- self.status_message.update(self.status_message_task_id, description=msg, visible=bool(msg))- yield- finally:- self.status_message.update(self.status_message_task_id, visible=False)-- @property- def fullscreen_layout(self) -> Layout:- if self.portrait:- return self.vertical_layout- return self.horizontal_layout-- def print_stats(self, stats: ScrapeStats) -> str:- if not self.manager.cli_args.print_stats:- return ""-- log_spacer()- logger.info("Printing Stats...\n")-- with capture_logs() as stream:- self._print_stats(stats)-- return stream.getvalue()-- def _print_stats(self, stats: ScrapeStats) -> None:-- elapsed = timedelta(seconds=int(time.monotonic() - stats.start_time))- total_data_written = ByteSize(self.file_progress.total_data_written).human_readable(decimal=True)-- logger.info("Run Stats:", extra={"color": "cyan"})- logger.info(f" Config file: {self.manager.config.source}")- logger.info(f" URLs source: {stats.source}")- logger.info(f" URLs: {stats.count:,}")- logger.info(f" URL groups: {len(stats.unique_groups):,}")- logger.info(f" Logs folder: {self.manager.config.settings.logs.log_folder}")- logger.info(f" Total runtime: {elapsed}")- logger.info(f" Total downloaded data: {total_data_written}")-- if stats.domain_stats:- log_spacer()- logger.info("URLs by domain:", extra={"color": "cyan"})- logger.info(" - " + "\n - ".join(map(str, stats.domain_stats.items())))-- log_spacer()- logger.info("Download Stats:", extra={"color": "cyan"})- logger.info(f" Downloaded: {self.download_progress.completed_files:,} files")- logger.info(f" Skipped (by config): {self.download_progress.skipped_files:,} files")- logger.info(f" Skipped (previously downloaded): {self.download_progress.previously_completed_files:,} files")- logger.info(f" Failed: {self.download_stats_progress.failed_files:,} files")-- log_spacer()- logger.info("Unsupported URLs Stats:", extra={"color": "cyan"})- logger.info(f" Sent to Jdownloader: {self.scrape_stats_progress.sent_to_jdownloader:,}")- logger.info(f" Skipped: {self.scrape_stats_progress.unsupported_urls_skipped:,}")-- hash_stats, dedupe_stats = self.manager.hasher.stats- self.print_hashing_stats(hash_stats)- self.print_dedupe_stats(dedupe_stats)-- log_spacer()- logger.info("Sort Stats:", extra={"color": "cyan"})- logger.info(f" Audios: {self.sort_progress.audio_count:,}")- logger.info(f" Images: {self.sort_progress.image_count:,}")- logger.info(f" Videos: {self.sort_progress.video_count:,}")- logger.info(f" Other Files: {self.sort_progress.other_count:,}")-- self.print_errors()-- def print_errors(self):-- _log_errors(self.scrape_stats_progress.return_totals(), self.download_stats_progress.return_totals())-- def print_hashing_stats(self, stats: HashingStats) -> None:- log_spacer()- logger.info("Checksum Stats:", extra={"color": "cyan"})- logger.info(f" Newly hashed: {stats.new_hashed:,} files")- logger.info(f" Previously hashed: {stats.prev_hashed:,} files")-- def print_dedupe_stats(self, stats: DedupeStats) -> None:- log_spacer()- logger.info("Dupe Stats:", extra={"color": "cyan"})- logger.info(f" Deleted (duplicates of previous downloads): {stats.deleted:,} files")- logger.info(f" Errors: {stats.total - stats.deleted:,} files")---def _log_errors(scrape_errors: Sequence[UiFailureTotal], download_errors: Sequence[UiFailureTotal]) -> None:- error_codes = (error.code for error in (*scrape_errors, *download_errors) if error.code is not None)-- try:- padding = len(str(max(error_codes)))- except ValueError:- padding = 0-- for title, errors in (- ("Scrape Failures:", scrape_errors),- ("Download Failures:", download_errors),- ):- log_spacer()- logger.info(title, extra={"color": "cyan"})- if not errors:- logger.info(f" {'None':>{padding}}", extra={"color": "green"})- continue-- for error in errors:- error_code = error.code if error.code is not None else ""- logger.info(- f" {error_code:>{padding}}{' ' if padding else ''}{error.msg}: {error.total:,}", extra={"color": "red"}- )diff --git a/cyberdrop_dl/ui/program_ui.py b/cyberdrop_dl/program_ui.pysimilarity index 100%rename from cyberdrop_dl/ui/program_ui.pyrename to cyberdrop_dl/program_ui.pydiff --git a/cyberdrop_dl/progress/__init__.py b/cyberdrop_dl/progress/__init__.pyindex b62629ca3..ef0bfa502 100644--- a/cyberdrop_dl/progress/__init__.py+++ b/cyberdrop_dl/progress/__init__.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport contextlibimport dataclasses+import sysfrom abc import ABC, abstractmethodfrom contextvars import ContextVarfrom typing import TYPE_CHECKING, Self@@ -92,7 +93,7 @@ class LiveUI(ABC):def __rich__(self) -> RenderableType: ...@contextlib.contextmanager- def __call__(self, *, transient: bool = False) -> Generator[None]:+ def __call__(self, *, transient: bool = True) -> Generator[None]:if self.disabled:yield Nonereturn@@ -101,9 +102,10 @@ class LiveUI(ABC):try:with Live(refresh_per_second=REFRESH_RATE.get(),- auto_refresh=True,+ auto_refresh="pytest" not in sys.modules,+ screen=transient,transient=transient,- get_renderable=self.__rich__,+ get_renderable=self.__rich__ if "pytest" not in sys.modules else None,):yieldfinally:diff --git a/cyberdrop_dl/progress/overflow.py b/cyberdrop_dl/progress/overflow.pyindex fea91faa5..931bb35c0 100644--- a/cyberdrop_dl/progress/overflow.py+++ b/cyberdrop_dl/progress/overflow.py@@ -12,6 +12,8 @@ from rich.progress import Task, TaskIDfrom cyberdrop_dl.progress import DictProgressif TYPE_CHECKING:+ from collections.abc import Callable+from rich.progress import ProgressColumn, Task, TaskID_COLOR: str = "plum3"@@ -54,12 +56,19 @@ class OverFlow:class OverFlowPanel:unit: ClassVar[str]- def __init__(self, *columns: ProgressColumn | str, max_rows: int, expand: bool = True) -> None:+ def __init__(+ self,+ *columns: ProgressColumn | str,+ max_rows: int,+ expand: bool = True,+ get_queue: Callable[[], int] = lambda: 0,+ ) -> None:self.max_rows: int = max_rowsself._progress: Final[DictProgress] = DictProgress(*columns, expand=expand)self._overflow: Final[OverFlow] = OverFlow(self.unit)self._invisible_rows: Final[deque[TaskID]] = deque()self._visible_rows: int = 0+ self.get_queue: Callable[[], int] = get_queueself._panel: Final[Panel] = Panel(Group(self._progress, self._overflow),title=type(self).__name__.removesuffix("Panel"),@@ -70,6 +79,7 @@ class OverFlowPanel:@finaldef __rich__(self) -> Panel:self._overflow.count = len(self._progress) - self._visible_rows+ self._overflow.queued = self.get_queue()return self._panel@finaldiff --git a/cyberdrop_dl/progress/scraping/downloads.py b/cyberdrop_dl/progress/scraping/downloads.pyindex b9659c934..7a585ce72 100644--- a/cyberdrop_dl/progress/scraping/downloads.py+++ b/cyberdrop_dl/progress/scraping/downloads.py@@ -10,6 +10,7 @@ from pathlib import Pathfrom typing import TYPE_CHECKING, ClassVar, Final, finalfrom rich.jupyter import JupyterMixin+from rich.markup import escapefrom rich.measure import Measurementfrom rich.progress import (BarColumn,@@ -140,7 +141,13 @@ class DownloadsPanel(OverFlowPanel):self._total_bytes = 0@contextlib.contextmanager- def download_hls(self, filename: str, /, segments: float | None = None) -> Generator[None]:+ def download_hls(+ self,+ filename: str,+ /,+ domain: str,+ segments: float,+ ) -> Generator[None]:# For HLS downloads, we use 2 different tasks. One on a hidden progress to track the downloaded bytes# and one on the user facing progress to track the number of downloaded segments (with a known total)# We create both at the same time and smuggle the bytes task as a field of the segments task@@ -148,7 +155,8 @@ class DownloadsPanel(OverFlowPanel):task_id = self._hls_progress.add_task("", total=None, visible=False)filename = str(filename).rsplit("/", 1)[-1]- segments_task = self._add_task(filename, segments)+ desc = escape((f"({domain.upper()}) {filename}").encode().decode("ascii", errors="ignore"))+ segments_task = self._add_task(desc, segments)bytes_task = self._hls_progress[task_id]self._progress.update(segments_task.id, HLS=bytes_task)token = _current_hls_task.set(segments_task.id)@@ -159,9 +167,16 @@ class DownloadsPanel(OverFlowPanel):self._hls_progress.remove_task(task_id)_current_hls_task.reset(token)- def download_file(self, description: object, /, total: float | None = None) -> ProgressHook:+ def download_file(+ self,+ description: object,+ /,+ domain: str,+ total: float | None,+ ) -> ProgressHook:filename = str(description).rsplit("/", 1)[-1]- task = self._add_task(filename, total)+ desc = escape((f"({domain.upper()}) {filename}").encode().decode("ascii", errors="ignore"))+ task = self._add_task(desc, total)def advance(amount: int = 1) -> None:self._total_bytes += amount@@ -204,7 +219,7 @@ class DownloadsPanel(OverFlowPanel):async def download_file(filename: str) -> None:size = random.randint(int(1e2), int(1e9))- hook = self.download_file(filename, size)+ hook = self.download_file(filename, "example.com", size)await download(hook, size)async def download_hls(filename: str) -> None:@@ -219,7 +234,7 @@ class DownloadsPanel(OverFlowPanel):finally:segments_sem.release()- with self.download_hls(filename, n_segments):+ with self.download_hls(filename, "example.com", n_segments):async with asyncio.TaskGroup() as tg:for _ in range(n_segments):await segments_sem.acquire()diff --git a/cyberdrop_dl/progress/scraping/panel.py b/cyberdrop_dl/progress/scraping/panel.pyindex 22c0ffffe..2ffcf62ce 100644--- a/cyberdrop_dl/progress/scraping/panel.py+++ b/cyberdrop_dl/progress/scraping/panel.py@@ -27,7 +27,7 @@ _generate_unique_id = itertools.count(1).__next__@final@dataclasses.dataclass(slots=True, frozen=True)class StatusMessage:- description: Text | str = f"Running cyberdrop-dl [blue]v{__version__}[/blue]"+ description: Text | str = f" cyberdrop-dl [blue]v{__version__}[/blue]"_messages: dict[int, tuple[Spinner, Text]] = dataclasses.field(init=False, default_factory=dict)_cols: Columns = dataclasses.field(init=False, default_factory=Columns)@@ -74,7 +74,7 @@ class ScrapingPanel(OverFlowPanel):@contextlib.contextmanagerdef new(self, url: object) -> Generator[None]:- task = self._add_task(str(url))+ task = self._add_task(escape(str(url).encode().decode("ascii", errors="ignore")))try:yieldfinally:diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex 300908879..00e908cbf 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -7,7 +7,6 @@ import datetimeimport loggingimport reimport time-from contextvars import ContextVarfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Literal, Self, TypeVar@@ -25,6 +24,7 @@ from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler, WordPressMediafrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItemfrom cyberdrop_dl.exceptions import JDownloaderError, NoExtensionErrorfrom cyberdrop_dl.logs import log_spacer+from cyberdrop_dl.progress.scraping import ScrapingUIfrom cyberdrop_dl.utils import filepathfrom cyberdrop_dl.utils.utilities import get_download_path, remove_trailing_slash@@ -45,8 +45,6 @@ logger = logging.getLogger(__name__)REGEX_LINKS = re.compile(r"(?:http.*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]))")-CURRENT_SCRAPE_DONE: ContextVar[asyncio.Event] = ContextVar("CURRENT_SCRAPE_DONE")-def _filter_by_date(scrape_item: ScrapeItem, before: datetime.date | None, after: datetime.date | None) -> bool:skip = False@@ -129,12 +127,24 @@ class ScrapeMapper:_seen_urls: set[AbsoluteHttpURL] = dataclasses.field(init=False, default_factory=set)_crawlers_disabled_at_runtime: set[str] = dataclasses.field(init=False, default_factory=set)_factory: CrawlerFactory = dataclasses.field(init=False)+ tui: ScrapingUI = dataclasses.field(init=False, default_factory=ScrapingUI)+ _done: asyncio.Event = dataclasses.field(init=False, default_factory=asyncio.Event)++ def _scrape_queue(self) -> int:+ return sum(f.waiting_items for f in self._factory)++ def _download_queue(self):+ total = sum(f.downloader.waiting_items for f in self._factory)+ self.tui.files.stats.queued = total+ return totaldef __post_init__(self) -> None:self._direct_http = DirectHttpFile(self.manager)self._jdownloader = JDownloader.from_manager(self.manager)self._real_debrid = RealDebridCrawler(self.manager)self._factory = CrawlerFactory(self.manager)+ self.tui.scrape.get_queue = self._scrape_queue+ self.tui.downloads.get_queue = self._download_queuedef create_task(self, coro: Coroutine[Any, Any, _T]) -> None:_ = self._task_groups.scrape.create_task(coro)@@ -155,30 +165,29 @@ class ScrapeMapper:@contextlib.asynccontextmanagerasync def __call__(self) -> AsyncGenerator[Self]:+ assert not self._done.is_set()_ = filepath.MAX_FILE_LEN.set(self.manager.config.global_settings.general.max_file_name_length)_ = filepath.MAX_FOLDER_LEN.set(self.manager.config.global_settings.general.max_folder_name_length)await self.manager.client_manager.load_cookie_files()## IMPORTANT: Order of each context matters!- async with (- self.manager.client_manager,- storage.monitor(self.manager.config.global_settings.general.required_free_space),- self.manager.logs.task_group,- self._task_groups.downloads,- ):- done = asyncio.Event()- token = CURRENT_SCRAPE_DONE.set(done)- try:- async with self._task_groups.scrape:- self.manager.scrape_mapper = self-- yield self-- finally:- # The done event signals that all scraping is done, but there may still be downloads pending- done.set()- CURRENT_SCRAPE_DONE.reset(token)+ with self.tui():+ async with (+ self.manager.client_manager,+ storage.monitor(self.manager.config.global_settings.general.required_free_space),+ self.manager.logs.task_group,+ self._task_groups.downloads,+ ):+ try:+ async with self._task_groups.scrape:+ self.manager.scrape_mapper = self++ yield self++ finally:+ # The done event signals that all scraping is done, but there may still be downloads pending+ self._done.set()async def run(self) -> ScrapeStats:self._init_crawlers()@@ -197,10 +206,10 @@ class ScrapeMapper:source_name, source = _source(self.manager)async with contextlib.aclosing(source) as items:stats = ScrapeStats(source_name)- done = CURRENT_SCRAPE_DONE.get()async def wait_until_scrape_is_done() -> None:- _ = await done.wait()+ _ = await self._done.wait()+ self.tui.hide_scrape_panel()stats.url_count.update((crawler.DOMAIN, count) for crawler in self._factory if (count := len(crawler._scraped_items)))@@ -266,12 +275,12 @@ class ScrapeMapper:else:success = True- self.manager.progress_manager.scrape_stats_progress.add_unsupported(sent_to_jdownloader=success)+ self.tui.scrape_errors.add_unsupported(sent_to_jdownloader=success)returnlogger.warning(f"Unsupported URL: {scrape_item.url}")self.manager.logs.write_unsupported(scrape_item.url, scrape_item.parents[0] if scrape_item.parents else None)- self.manager.progress_manager.scrape_stats_progress.add_unsupported()+ self.tui.scrape_errors.add_unsupported()def _should_scrape(self, scrape_item: ScrapeItem) -> bool:if scrape_item.url in self._seen_urls:diff --git a/cyberdrop_dl/ui/progress/__init__.py b/cyberdrop_dl/ui/progress/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/cyberdrop_dl/ui/progress/deque_progress.py b/cyberdrop_dl/ui/progress/deque_progress.pydeleted file mode 100644index ec1645501..000000000--- a/cyberdrop_dl/ui/progress/deque_progress.py+++ /dev/null@@ -1,118 +0,0 @@-from __future__ import annotations--from abc import ABC, abstractmethod-from collections import deque-from itertools import islice-from typing import TYPE_CHECKING--from rich.console import Group-from rich.panel import Panel-from rich.progress import Progress, TaskID--if TYPE_CHECKING:- from collections.abc import Sequence---def adjust_title(s: str, length: int = 40, placeholder: str = "...") -> str:- """Collapse and truncate or pad the given string to fit in the given length."""- return f"{s[: length - len(placeholder)]}{placeholder}" if len(s) >= length else s.ljust(length)---class DequeProgress(ABC):- _progress: Progress- type_str: str = "files"- color = "plum3"- progress_str = "[{color}]{description}"- overflow_str = "[{color}]... and {number:,} other {type_str}"- queue_str = "[{color}]... and {number:,} {type_str} in {title} queue"-- def __init__(self, title: str, visible_tasks_limit: int) -> None:- self.title = title- self.title_lower = title.lower()- self._overflow = Progress("[progress.description]{task.description}")- self._queue = Progress("[progress.description]{task.description}")- self._progress_group = Group(self._progress, self._overflow, self._queue)-- self._overflow_task_id = self._overflow.add_task(- self.overflow_str.format(color=self.color, number=0, type_str=self.type_str),- visible=False,- )- self._queue_task_id = self._queue.add_task(- self.queue_str.format(color=self.color, number=0, type_str=self.type_str, title=self.title_lower),- visible=False,- )- self._tasks: deque[TaskID] = deque()- self._tasks_visibility_limit = visible_tasks_limit-- @abstractmethod- def get_queue_length(self) -> int: ...-- @property- def visible_tasks(self) -> Sequence[TaskID]:- if len(self._tasks) > self._tasks_visibility_limit:- return [self._tasks[i] for i in range(self._tasks_visibility_limit)]- return self._tasks-- @property- def invisible_tasks(self) -> Sequence[TaskID]:- return list(islice(self._tasks, self._tasks_visibility_limit, None))-- @property- def invisible_tasks_len(self) -> int:- """Faster to compute than `len(self.invisible_tasks)`"""- return max(0, len(self._tasks) - self._tasks_visibility_limit)-- def has_visible_capacity(self) -> bool:- return len(self._tasks) < self._tasks_visibility_limit-- def get_renderable(self) -> Panel:- """Returns the progress bar."""- return Panel(self._progress_group, title=self.title, border_style="green", padding=(1, 1))-- def add_task(self, description: str, total: float | None = None) -> TaskID:- """Adds a new task to the progress bar."""- task_id = self._progress.add_task(- self.progress_str.format(color=self.color, description=description),- total=total,- visible=self.has_visible_capacity(),- )- self._tasks.append(task_id)- self.redraw()- return task_id-- def remove_task(self, task_id: TaskID) -> None:- """Removes a task from the progress bar."""- if task_id not in self._tasks:- msg = "Task ID not found"- raise ValueError(msg)-- self._tasks.remove(task_id)- self._progress.remove_task(task_id)- self.redraw()-- def redraw(self) -> None:- """Redraws the progress bar."""- for task in self.visible_tasks:- self._progress.update(task, visible=True)-- invisible_tasks_len = self.invisible_tasks_len-- self._overflow.update(- self._overflow_task_id,- description=self.overflow_str.format(- color=self.color,- number=invisible_tasks_len,- type_str=self.type_str,- ),- visible=invisible_tasks_len > 0,- )-- queue_length = self.get_queue_length()-- self._queue.update(- self._queue_task_id,- description=self.queue_str.format(- color=self.color, number=queue_length, type_str=self.type_str, title=self.title_lower- ),- visible=queue_length > 0,- )diff --git a/cyberdrop_dl/ui/progress/downloads_progress.py b/cyberdrop_dl/ui/progress/downloads_progress.pydeleted file mode 100644index 3a60017a8..000000000--- a/cyberdrop_dl/ui/progress/downloads_progress.py+++ /dev/null@@ -1,101 +0,0 @@-from __future__ import annotations--from typing import TYPE_CHECKING--from rich.console import Group-from rich.panel import Panel-from rich.progress import BarColumn, Progress--if TYPE_CHECKING:- from cyberdrop_dl.managers.manager import Manager---class DownloadsProgress:- """Class that keeps track of completed, skipped and failed files."""-- def __init__(self, manager: Manager) -> None:- self.manager = manager- self.progress = Progress(- "[progress.description]{task.description}",- BarColumn(bar_width=None),- "[progress.percentage]{task.percentage:>6.2f}%",- "━",- "{task.completed:,}",- )- self.progress_group = Group(self.progress)-- self.total_files = 0- self.completed_files_task_id = self.progress.add_task("[green]Completed", total=0)- self.completed_files = 0- self.previously_completed_files_task_id = self.progress.add_task("[yellow]Previously Downloaded", total=0)- self.previously_completed_files = 0- self.skipped_files_task_id = self.progress.add_task("[yellow]Skipped By Configuration", total=0)- self.skipped_files = 0- self.queued_files_task_id = self.progress.add_task("[cyan]Queued", total=0)- self.queued_files = 0- self.failed_files_task_id = self.progress.add_task("[red]Failed", total=0)- self.failed_files = 0- self.panel = Panel(- self.progress_group,- title="Files",- border_style="green",- padding=(1, 1),- subtitle=f"Total Files: [white]{self.total_files:,}",- )- self.simple_progress = Progress(- "[progress.description]{task.description}",- BarColumn(bar_width=None),- "[progress.percentage]{task.percentage:>6.2f}%",- "━",- "{task.completed:,}",- )- self.simple_progress_task_id = self.simple_progress.add_task("Completed", total=0)-- @property- def simple_completed(self):- return self.total_files - self.queued_files-- def get_progress(self) -> Panel:- """Returns the progress bar."""- return self.panel-- def update_total(self, increase_total: bool = True) -> None:- """Updates the total number of files to be downloaded."""- if increase_total:- self.total_files = self.total_files + 1- self.progress.update(self.completed_files_task_id, total=self.total_files)- self.progress.update(self.previously_completed_files_task_id, total=self.total_files)- self.progress.update(self.skipped_files_task_id, total=self.total_files)- self.progress.update(self.failed_files_task_id, total=self.total_files)- self.progress.update(self.queued_files_task_id, total=self.total_files)- self.simple_progress.update(- self.simple_progress_task_id, total=self.total_files, completed=self.simple_completed- )- self.panel.subtitle = f"Total Files: [white]{self.total_files:,}"-- def add_completed(self) -> None:- """Adds a completed file to the progress bar."""- self.progress.advance(self.completed_files_task_id, 1)- self.completed_files += 1-- def add_previously_completed(self, increase_total: bool = True) -> None:- """Adds a previously completed file to the progress bar."""- if increase_total:- self.update_total()- self.previously_completed_files += 1- self.progress.advance(self.previously_completed_files_task_id, 1)-- def add_skipped(self) -> None:- """Adds a skipped file to the progress bar."""- self.progress.advance(self.skipped_files_task_id, 1)- self.skipped_files += 1-- def add_failed(self) -> None:- """Adds a failed file to the progress bar."""- self.progress.advance(self.failed_files_task_id, 1)- self.failed_files += 1-- def update_queued(self, number: int) -> None:- """Adds a queed file to the progress bar."""- self.queued_files = number- self.progress.update(self.queued_files_task_id, completed=self.queued_files)diff --git a/cyberdrop_dl/ui/progress/file_progress.py b/cyberdrop_dl/ui/progress/file_progress.pydeleted file mode 100644index 13409d468..000000000--- a/cyberdrop_dl/ui/progress/file_progress.py+++ /dev/null@@ -1,79 +0,0 @@-from __future__ import annotations--from typing import TYPE_CHECKING--from rich.markup import escape-from rich.progress import (- BarColumn,- DownloadColumn,- Progress,- SpinnerColumn,- TaskID,- TimeRemainingColumn,- TransferSpeedColumn,-)--from cyberdrop_dl.ui.progress.deque_progress import DequeProgress, adjust_title--if TYPE_CHECKING:- from cyberdrop_dl.managers.manager import Manager---class FileProgress(DequeProgress):- """Class that manages the download progress of individual files."""-- def __init__(self, manager: Manager) -> None:- self.manager = manager- progress_colums = (SpinnerColumn(), "[progress.description]{task.description}", BarColumn(bar_width=None))- visible_tasks_limit: int = 10- horizontal_columns = (- *progress_colums,- "[progress.percentage]{task.percentage:>6.2f}%",- "━",- DownloadColumn(),- "━",- TransferSpeedColumn(),- "━",- TimeRemainingColumn(),- )- vertical_columns = (*progress_colums, DownloadColumn(), "━", TransferSpeedColumn())- use_columns = horizontal_columns- if manager.cli_args.portrait:- use_columns = vertical_columns- self._progress = Progress(*use_columns)- self.total_data_written = 0- super().__init__("Downloads", visible_tasks_limit)-- def get_queue_length(self) -> int:- """Returns the number of tasks in the downloader queue."""- total = 0- unique_crawler_ids = set()- for crawler in self.manager.scrape_mapper._factory:- crawler_id = id(crawler) # Only count each instance of the crawler once- if crawler_id in unique_crawler_ids:- continue- unique_crawler_ids.add(crawler_id)- total += getattr(crawler.downloader, "waiting_items", 0)-- return total-- def add_task(self, *, domain: str, filename: str, expected_size: int | None = None) -> TaskID: # type: ignore[reportIncompatibleMethodOverride]- """Adds a new task to the progress bar."""- filename = filename.split("/")[-1].encode("ascii", "ignore").decode().strip()- description = escape(adjust_title(filename, length=40))- if not self.manager.progress_manager.portrait:- description = f"({domain.upper()}) {description}"- return super().add_task(description, expected_size)-- def advance_file(self, task_id: TaskID, amount: int) -> None:- """Advances the progress of the given task by the given amount."""- self.total_data_written += amount- self._progress.advance(task_id, amount)-- def get_speed(self, task_id: TaskID) -> float:- if task_id not in self._tasks:- msg = "Task ID not found"- raise ValueError(msg)-- task = self._progress._tasks[task_id]- return task.finished_speed or task.speed or 0diff --git a/cyberdrop_dl/ui/progress/hash_progress.py b/cyberdrop_dl/ui/progress/hash_progress.pydeleted file mode 100644index 625044a72..000000000--- a/cyberdrop_dl/ui/progress/hash_progress.py+++ /dev/null@@ -1,125 +0,0 @@-from __future__ import annotations--import contextlib-from pathlib import Path-from typing import TYPE_CHECKING--from pydantic import ByteSize-from rich.console import Group-from rich.markup import escape-from rich.panel import Panel-from rich.progress import BarColumn, Progress, TaskID--if TYPE_CHECKING:- from cyberdrop_dl.managers.manager import Manager---def _generic_progress() -> Progress:- return Progress("[progress.description]{task.description}", BarColumn(bar_width=None), "{task.completed:,}")---class HashProgress:- """Class that keeps track of hashed files."""-- def __init__(self, manager: Manager) -> None:- self.manager = manager- self._hash_progress = _generic_progress()- self._remove_progress = _generic_progress()- self._match_progress = _generic_progress()- self._file_info = Progress("{task.description}")- self._base_dir: Path | None = None-- # hashing- self._computed_hashes = self._prev_hashed = 0- self.hash_progress_group = Group(self._file_info, self._hash_progress)-- self._tasks: dict[str, TaskID] = {}-- def add_hashed_task(hash_type: str) -> None:- desc = "[green]Hashed " + escape(f"[{hash_type}]")- self._tasks[hash_type] = self._hash_progress.add_task(desc, total=None)-- add_hashed_task("xxh128")- if manager.config.settings.dupe_cleanup_options.add_md5_hash:- add_hashed_task("md5")- if manager.config.settings.dupe_cleanup_options.add_sha256_hash:- add_hashed_task("sha256")-- self.prev_hashed_files_task_id = self._hash_progress.add_task("[green]Previously Hashed", total=None)-- self._base_dir_task_id = self._file_info.add_task("")- self._file_task_id = self._file_info.add_task("")-- # remove- self.removed_files = 0- self.removed_progress_group = Group(self._match_progress, self._remove_progress)- self.removed_files_task_id = self._remove_progress.add_task(- "[green]Removed From Downloaded Files",- total=None,- )-- @property- def hashed_files(self) -> int:- return int(self._computed_hashes / len(self._tasks))-- @property- def prev_hashed_files(self) -> int:- return int(self._prev_hashed / len(self._tasks))-- def get_renderable(self) -> Panel:- """Returns the progress bar."""- return Panel(- self.hash_progress_group,- title="Hashing",- border_style="green",- padding=(1, 1),- )-- def get_removed_progress(self) -> Panel:- """Returns the progress bar."""- return Panel(self.removed_progress_group, border_style="green", padding=(1, 1))-- @contextlib.contextmanager- def currently_hashing_dir(self, path: Path):- self._base_dir = path- desc = "[green]Base dir: [blue]" + escape(f"{self._base_dir}")- self._file_info.update(self._base_dir_task_id, description=desc)- try:- yield- finally:- self._base_dir = None- self._file_info.update(self._base_dir_task_id, description="")-- def update_currently_hashing(self, file: Path) -> None:- if not self._base_dir:- return- file_size = ByteSize(Path(file).stat().st_size)- size_text = file_size.human_readable(decimal=True)- path = file.relative_to(self._base_dir)- desc = "[green]Current file: [blue]" + escape(f"{path}") + f" [green]({size_text})"- self._file_info.update(self._file_task_id, description=desc)-- def add_new_completed_hash(self, hash_type: str) -> None:- """Adds a completed file to the progress bar."""- self._hash_progress.advance(self._tasks[hash_type], 1)- self._computed_hashes += 1-- def add_prev_hash(self) -> None:- """Adds a completed file to the progress bar."""- self._hash_progress.advance(self.prev_hashed_files_task_id, 1)- self._prev_hashed += 1-- def add_removed_file(self) -> None:- """Adds a removed file to the progress bar."""- self._remove_progress.advance(self.removed_files_task_id, 1)- self.removed_files += 1-- def reset(self):- """Resets the progress bar."""- for task in self._tasks.values():- self._hash_progress.reset(task)- self._hash_progress.reset(self.prev_hashed_files_task_id)- self._computed_hashes = self._prev_hashed = 0-- self._remove_progress.reset(self.removed_files_task_id)- self.removed_files = 0diff --git a/cyberdrop_dl/ui/progress/scraping_progress.py b/cyberdrop_dl/ui/progress/scraping_progress.pydeleted file mode 100644index 8b36e27d4..000000000--- a/cyberdrop_dl/ui/progress/scraping_progress.py+++ /dev/null@@ -1,46 +0,0 @@-from __future__ import annotations--from typing import TYPE_CHECKING--from rich.progress import Progress, SpinnerColumn, TaskID--from cyberdrop_dl.ui.progress.deque_progress import DequeProgress--if TYPE_CHECKING:- from yarl import URL-- from cyberdrop_dl.managers.manager import Manager---class ScrapingProgress(DequeProgress):- """Class that manages the download progress of individual files."""-- type_str = "URLs"-- def __init__(self, manager: Manager) -> None:- self.manager = manager- self._progress = Progress(SpinnerColumn(), "[progress.description]{task.description}")- visible_tasks_limit: int = 5- super().__init__("Scraping", visible_tasks_limit)-- def get_queue_length(self) -> int:- """Returns the number of tasks in the scraper queue."""- total = 0- unique_crawler_ids = set()- for crawler in self.manager.scrape_mapper._factory:- crawler_id = id(crawler) # Only count each instance of the crawler once- if crawler_id in unique_crawler_ids:- continue- unique_crawler_ids.add(crawler_id)- total += crawler.waiting_items-- return total-- def redraw(self, passed: bool = False) -> None:- super().redraw()- if not passed:- self.manager.progress_manager.file_progress.redraw()-- def add_task(self, url: URL) -> TaskID: # type: ignore[reportIncompatibleMethodOverride]- """Adds a new task to the progress bar."""- return super().add_task(str(url))diff --git a/cyberdrop_dl/ui/progress/sort_progress.py b/cyberdrop_dl/ui/progress/sort_progress.pydeleted file mode 100644index 5d1aca7f8..000000000--- a/cyberdrop_dl/ui/progress/sort_progress.py+++ /dev/null@@ -1,71 +0,0 @@-from __future__ import annotations--from typing import TYPE_CHECKING--from rich.markup import escape-from rich.panel import Panel-from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID--from cyberdrop_dl.ui.progress.deque_progress import DequeProgress, adjust_title--if TYPE_CHECKING:- from cyberdrop_dl.managers.manager import Manager---class SortProgress(DequeProgress):- """Class that keeps track of sorted files."""-- type_str = "Folders"-- def __init__(self, visible_tasks_limit: int, manager: Manager) -> None:- """Sorter to track the progress of folders being sorted.-- Should work similar to the file_progress but for folders, with a percentage and progress bar for the files within the folders"""- self.manager = manager- self._progress = Progress(- SpinnerColumn(),- "[progress.description]{task.description}",- BarColumn(bar_width=None),- "[progress.percentage]{task.percentage:>6.2f}%",- "━",- "{task.completed}/{task.total} files",- )- super().__init__("Sort", visible_tasks_limit)-- # counts- self.queue_length = self.audio_count = self.video_count = self.image_count = self.other_count = 0-- def get_queue_length(self) -> int:- return self.queue_length-- def get_renderable(self) -> Panel:- """Returns the progress bar."""- return Panel(- self._progress_group,- title="Sorting Downloads",- border_style="green",- padding=(1, 1),- )-- def add_task(self, folder: str, expected_size: int | None) -> TaskID:- """Adds a new task to the progress bar."""- # description = f'Sorting {folder}'- description = folder.encode("ascii", "ignore").decode().strip()- description = escape(adjust_title(description))- return super().add_task(description, expected_size)-- def advance_folder(self, task_id: TaskID, amount: int = 1) -> None:- """Advances the progress of the given task by the given amount."""- self._progress.advance(task_id, amount)-- def increment_audio(self) -> None:- self.audio_count += 1-- def increment_video(self) -> None:- self.video_count += 1-- def increment_image(self) -> None:- self.image_count += 1-- def increment_other(self) -> None:- self.other_count += 1diff --git a/cyberdrop_dl/ui/progress/statistic_progress.py b/cyberdrop_dl/ui/progress/statistic_progress.pydeleted file mode 100644index b45e3c4ec..000000000--- a/cyberdrop_dl/ui/progress/statistic_progress.py+++ /dev/null@@ -1,186 +0,0 @@-from __future__ import annotations--import contextlib-import functools-from typing import NamedTuple--from rich.console import Group-from rich.panel import Panel-from rich.progress import BarColumn, Progress, TaskID--FAILURE_OVERRIDES = {- "ClientConnectorCertificateError": "Client Connector Certificate Error",- "ClientConnectorDNSError": "Client Connector DNS Error",- "ClientConnectorError": "Client Connector Error",- "ClientConnectorSSLError": "Client Connector SSL Error",- "ClientHttpProxyError": "Client HTTP Proxy Error",- "ClientPayloadError": "Client Payload Error",- "ClientProxyConnectionError": "Client Proxy Connection Error",- "ConnectionTimeoutError": "Connection Timeout",- "ContentTypeError": "Content Type Error",- "InvalidURL": "Invalid URL",- "InvalidUrlClientError": "Invalid URL Client Error",- "InvalidUrlRedirectClientError": "Invalid URL Redirect",- "NonHttpUrlRedirectClientError": "Non HTTP URL Redirect",- "RedirectClientError": "Redirect Error",- "ServerConnectionError": "Server Connection Error",- "ServerDisconnectedError": "Server Disconnected",- "ServerFingerprintMismatch": "Server Fingerprint Mismatch",- "ServerTimeoutError": "Server Timeout Error",- "SocketTimeoutError": "Socket Timeout Error",-}---class TaskInfo(NamedTuple):- id: TaskID- description: str- completed: float- total: float | None- progress: float---class UiFailureTotal(NamedTuple):- full_msg: str- total: int- code: int | None- msg: str-- @classmethod- def from_pair(cls, full_msg: str, total: int) -> UiFailureTotal:- parts = full_msg.split(" ", 1)- if len(parts) > 1 and parts[0].isdigit():- error_code, msg = parts- return cls(full_msg, total, int(error_code), msg)- return cls(full_msg, total, None, full_msg)---def get_tasks_info_sorted(progress: Progress) -> tuple[list[TaskInfo], bool]:- tasks = [- TaskInfo(- id=task.id,- description=task.description,- completed=task.completed,- total=task.total,- progress=(task.completed / task.total if task.total else 0),- )- for task in progress.tasks- ]-- tasks_sorted = sorted(tasks, key=lambda x: x.completed, reverse=True)- were_sorted = tasks == tasks_sorted- return tasks_sorted, were_sorted---class StatsProgress:- """Base class that keeps track of failures and reasons."""-- title = "Download Failures"-- def __init__(self) -> None:- self.progress = Progress(- "[progress.description]{task.description}",- BarColumn(bar_width=None),- "[progress.percentage]{task.percentage:>6.2f}%",- "━",- "{task.completed:,}",- )- self.progress_group = Group(self.progress)- self.failure_types: dict[str, TaskID] = {}- self.failed_files = 0- self.panel = Panel(- self.progress_group,- title=self.title,- border_style="green",- padding=(1, 1),- subtitle=self.subtitle,- )-- @property- def subtitle(self) -> str:- return f"Total {self.title}: [white]{self.failed_files:,}"-- def get_progress(self) -> Panel:- """Returns the progress bar."""- return self.panel-- def update_total(self, total: int) -> None:- """Updates the total number download failures."""- self.panel.subtitle = self.subtitle- for key in self.failure_types:- self.progress.update(self.failure_types[key], total=total)-- tasks_sorted, were_sorted = get_tasks_info_sorted(self.progress)- if not were_sorted:- self.sort_tasks(tasks_sorted)-- def sort_tasks(self, tasks_sorted: list[TaskInfo]) -> None:- for task_id in [task.id for task in tasks_sorted]:- self.progress.remove_task(task_id)-- for task in tasks_sorted:- self.failure_types[task.description] = self.progress.add_task(- task.description,- total=task.total,- completed=task.completed, # type: ignore- )-- def add_failure(self, failure: str) -> None:- """Adds a failed file to the progress bar."""- self.failed_files += 1- key = get_pretty_failure(failure)- task_id = self.failure_types.get(key)- if task_id is not None:- self.progress.advance(task_id)- else:- self.failure_types[key] = self.progress.add_task(key, total=self.failed_files, completed=1)- self.update_total(self.failed_files)-- def return_totals(self) -> list[UiFailureTotal]:- """Returns the total number of failed sites and reasons."""- failures = {}- for key, task_id in self.failure_types.items():- task = next(task for task in self.progress.tasks if task.id == task_id)- failures[key] = task.completed- return sorted(UiFailureTotal.from_pair(*f) for f in failures.items())---class DownloadStatsProgress(StatsProgress):- """Class that keeps track of download failures and reasons."""---class ScrapeStatsProgress(StatsProgress):- """Class that keeps track of scraping failures and reasons."""-- title = "Scrape Failures"-- def __init__(self) -> None:- super().__init__()- self.unsupported_urls = 0- self.sent_to_jdownloader = 0- self.unsupported_urls_skipped = 0-- def add_unsupported(self, sent_to_jdownloader: bool = False) -> None:- """Adds an unsupported url to the progress bar."""- self.unsupported_urls += 1- if sent_to_jdownloader:- self.sent_to_jdownloader += 1- else:- self.unsupported_urls_skipped += 1--[email protected]_cache-def get_pretty_failure(failure: str) -> str:- with contextlib.suppress(KeyError):- return FAILURE_OVERRIDES[failure]- return capitalize_words(failure)---def capitalize_words(text: str) -> str:- """Capitalize first letter of each word-- Unlike `str.capwords()`, this only caps the first letter of each word without modifying the rest of the word"""- return " ".join([capitalize_first_letter(word) for word in text.split()])---def capitalize_first_letter(word: str) -> str:- return word[0].capitalize() + word[1:]diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex f7e9b3c55..100670151 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -145,7 +145,7 @@ def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yarlogger.error(f"Scrape Failed: {link_to_show} ({error_log_msg.main_log_msg})", exc_info=exc_info)self.manager.logs.write_scrape_error(link_to_show, error_log_msg.csv_log_msg, origin)- self.manager.progress_manager.scrape_stats_progress.add_failure(error_log_msg.ui_failure)+ self.manager.scrape_mapper.tui.scrape_errors.add(error_log_msg.ui_failure)@overloaddiff --git a/tests/crawlers/test_xenforo.py b/tests/crawlers/test_xenforo.pyindex 03930229b..10736db48 100644--- a/tests/crawlers/test_xenforo.py+++ b/tests/crawlers/test_xenforo.py@@ -18,13 +18,8 @@ def _item(url: str) -> ScrapeItem:return ScrapeItem(url=AbsoluteHttpURL(url))-class MockProgress:- scraping_progress = None--manager = Manager()-scrape_item = _item("https://xenforo.com/community")-manager.progress_manager = MockProgress() # type: ignore+scrape_item = _item("https://xenforo.com/community") # type: ignorecrawler_instances = {crawler: crawler(manager) for crawler in crawlers.XF_CRAWLERS}TEST_CRAWLER = crawler_instances[crawlers.CelebForumCrawler]diff --git a/tests/test_startup.py b/tests/test_startup.pyindex dcf4c370b..27a270e37 100644--- a/tests/test_startup.py+++ b/tests/test_startup.py@@ -1,7 +1,7 @@import pytest+from cyberdrop_dl import program_uifrom cyberdrop_dl.__main__ import main-from cyberdrop_dl.ui import program_uidef test_startup(capsys: pytest.CaptureFixture[str]) -> None: