Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: use new scraping UI (#148)

NTFSvolume5e95ffa78beb
diff --git a/cyberdrop_dl/__main__.py b/cyberdrop_dl/__main__.py
index 7d6de78a2..8bd1ebcad 100644
--- a/cyberdrop_dl/__main__.py
+++ b/cyberdrop_dl/__main__.py
@@ -10,14 +10,13 @@ from typing import Annotated
from cyclopts import App, Parameter
-from cyberdrop_dl import __version__, aio, webhook
+from cyberdrop_dl import __version__, aio, program_ui, webhook
from cyberdrop_dl.cli import CLIargs
from cyberdrop_dl.config import Config
from cyberdrop_dl.logs import log_spacer, setup_console_logging, setup_file_logging
from cyberdrop_dl.managers.manager import AppData, Manager
from cyberdrop_dl.models.types import HttpURL
from cyberdrop_dl.scrape_mapper import ScrapeMapper
-from cyberdrop_dl.ui import program_ui
from cyberdrop_dl.utils import apprise, check_latest_pypi
from cyberdrop_dl.utils.sorting import Sorter
from cyberdrop_dl.utils.utilities import check_partials_and_empty_folders
@@ -33,14 +32,13 @@ async def _scrape(manager: Manager) -> None:
async with manager.database:
log_spacer()
logger.info("Starting CDL...")
- with manager.live_manager.get_main_live(stop=True):
- async with ScrapeMapper(manager)() as scrape_mapper:
- stats = await scrape_mapper.run()
+ async with ScrapeMapper(manager)() as scrape_mapper:
+ stats = await scrape_mapper.run()
log_spacer()
await _post_runtime(manager)
- stats_summary = manager.progress_manager.print_stats(stats)
+ stats_summary = manager.print_stats(stats)
log_spacer()
await check_latest_pypi()
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index c0eb17c12..ed6dda87a 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem
from cyberdrop_dl.managers.client_manager import ClientManager
from cyberdrop_dl.managers.manager import Manager
+ from cyberdrop_dl.progress import ProgressHook
logger = logging.getLogger(__name__)
@@ -102,13 +103,13 @@ class DownloadClient:
proceed, skip = await self.get_final_file_info(media_item, domain)
self.client_manager.check_content_length(resp.headers)
if skip:
- self.manager.progress_manager.download_progress.add_skipped()
+ self.manager.scrape_mapper.tui.files.stats.skipped += 1
return False
if not proceed:
if media_item.is_segment:
return True
logger.info(f"Skipping {media_item.url} as it has already been downloaded")
- self.manager.progress_manager.download_progress.add_previously_completed(False)
+ self.manager.scrape_mapper.tui.files.stats.previously_completed += 1
await self.process_completed(media_item, domain)
await self.handle_media_item_completion(media_item, downloaded=False)
@@ -127,21 +128,26 @@ class DownloadClient:
)
media_item.uploaded_at = last_modified
- task_id = media_item.task_id
- if task_id is None:
+ if media_item.is_segment:
+ hook = self.manager.scrape_mapper.tui.downloads.download_hls_seg()
+
+ else:
size = (media_item.filesize + resume_point) if media_item.filesize is not None else None
- task_id = self.manager.progress_manager.file_progress.add_task(
- domain=domain, filename=media_item.filename, expected_size=size
+ hook = self.manager.scrape_mapper.tui.downloads.download_file(
+ media_item.filename,
+ media_item.domain,
+ size,
)
- media_item.set_task_id(task_id)
- self.manager.progress_manager.file_progress.advance_file(task_id, resume_point)
+ if resume_point:
+ hook.advance(resume_point)
- await self._append_content(media_item, self._get_resp_reader(resp))
+ with hook:
+ await self._append_content(media_item, hook, self._get_resp_reader(resp))
return True
def _get_resp_reader(
- self, resp: aiohttp.ClientResponse | AbstractResponse
+ self, resp: aiohttp.ClientResponse | AbstractResponse[Any]
) -> AbstractResponse | aiohttp.StreamReader:
if isinstance(resp, AbstractResponse):
return resp
@@ -165,7 +171,7 @@ class DownloadClient:
async def _request_download(
self,
media_item: MediaItem,
- process_response: Callable[[aiohttp.ClientResponse | AbstractResponse], Coroutine[None, None, bool]],
+ process_response: Callable[[aiohttp.ClientResponse | AbstractResponse[Any]], Coroutine[None, None, bool]],
) -> bool:
download_url = media_item.debrid_link or media_item.url
@@ -200,13 +206,15 @@ class DownloadClient:
raise
async def _append_content(
- self, media_item: MediaItem, content: aiohttp.StreamReader | AbstractResponse[Any]
+ self,
+ media_item: MediaItem,
+ hook: ProgressHook,
+ content: aiohttp.StreamReader | AbstractResponse[Any],
) -> None:
"""Appends content to a file."""
- assert media_item.task_id is not None
check_free_space = storage.create_free_space_checker(media_item)
- check_download_speed = self.make_speed_checker(media_item)
+ check_download_speed = self.make_speed_checker(media_item, hook)
await check_free_space()
await self._pre_download_check(media_item)
@@ -216,7 +224,7 @@ class DownloadClient:
chunk_size = len(chunk)
await self.client_manager.speed_limiter.acquire(chunk_size)
await f.write(chunk)
- self.manager.progress_manager.file_progress.advance_file(media_item.task_id, chunk_size)
+ hook.advance(chunk_size)
check_download_speed()
await self._post_download_check(media_item)
@@ -232,15 +240,15 @@ class DownloadClient:
await aio.unlink(media_item.partial_file, missing_ok=True)
raise DownloadError(HTTPStatus.INTERNAL_SERVER_ERROR, message="File is empty")
- def make_speed_checker(self, media_item: MediaItem) -> Callable[[], None]:
+ def make_speed_checker(self, media_item: MediaItem, hook: ProgressHook) -> Callable[[], None]:
last_slow_speed_read = None
def check_download_speed() -> None:
nonlocal last_slow_speed_read
if not self.download_speed_threshold:
return
- assert media_item.task_id is not None
- speed = self.manager.progress_manager.file_progress.get_speed(media_item.task_id)
+
+ speed = hook.get_speed()
if speed > self.download_speed_threshold:
last_slow_speed_read = None
elif not last_slow_speed_read:
@@ -254,7 +262,7 @@ class DownloadClient:
"""Starts a file."""
if self.manager.config.settings.download_options.skip_download_mark_completed and not media_item.is_segment:
logger.info(f"Download removed {media_item.url} due to mark completed option")
- self.manager.progress_manager.download_progress.add_skipped()
+ self.manager.scrape_mapper.tui.files.stats.skipped += 1
# set completed path
await self.process_completed(media_item, domain)
return False
@@ -271,7 +279,7 @@ class DownloadClient:
logger.info(f"Download skipped {media_item.url} due to runtime restrictions")
await aio.unlink(media_item.path)
await self.mark_incomplete(media_item, domain)
- self.manager.progress_manager.download_progress.add_skipped()
+ self.manager.scrape_mapper.tui.files.stats.skipped += 1
return False
await self.process_completed(media_item, domain)
await self.handle_media_item_completion(media_item, downloaded=True)
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
index b986644f7..6c6bb8f24 100644
--- a/cyberdrop_dl/clients/flaresolverr.py
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -14,6 +14,7 @@ from multidict import CIMultiDict, CIMultiDictProxy
from cyberdrop_dl.data_structures import AbsoluteHttpURL
from cyberdrop_dl.exceptions import DDOSGuardError
+from cyberdrop_dl.progress.scraping import show_msg
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Mapping
@@ -148,9 +149,11 @@ class FlareSolverrClient:
params["postData"] = aiohttp.FormData(data)().decode()
async with self._request_lock:
- logger.debug(f"Making FlareSolverr request #{self._request_id()} with {params = }")
- async with self._aiohttp_session.post(self.url, json=params, **timeout) as response:
- return Response.from_dict(await response.json())
+ request_id = self._request_id()
+ with show_msg(f"Waiting for flaresolver [{request_id}]"):
+ logger.debug(f"Making FlareSolverr request #{request_id} with {params = }")
+ async with self._aiohttp_session.post(self.url, json=params, **timeout) as response:
+ return Response.from_dict(await response.json())
async def _create_session(self) -> None:
session_id = "cyberdrop-dl"
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 799dcbf5b..574798c7a 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -45,7 +45,6 @@ if TYPE_CHECKING:
import yarl
from bs4 import BeautifulSoup, Tag
from curl_cffi.requests.impersonate import BrowserTypeLiteral
- from rich.progress import TaskID
from cyberdrop_dl.managers.manager import Manager
@@ -366,15 +365,10 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
raise exc
@final
- @contextlib.contextmanager
- def new_task_id(self, url: AbsoluteHttpURL) -> Generator[TaskID]:
+ def new_task_id(self, url: AbsoluteHttpURL):
"""Creates a new task_id (shows the URL in the UI and logs)"""
self.log.info(f"Scraping {url}")
- task_id = self.manager.progress_manager.scraping_progress.add_task(url)
- try:
- yield task_id
- finally:
- self.manager.progress_manager.scraping_progress.remove_task(task_id)
+ return self.manager.scrape_mapper.tui.scrape.new(url)
@final
@staticmethod
@@ -482,7 +476,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
current_referer, downloaded = await self.manager.database.history.check_complete(self.DOMAIN, db_path)
if downloaded:
logger.info("Skipping %s as it has already been downloaded", url)
- self.manager.progress_manager.download_progress.add_previously_completed()
+ self.manager.scrape_mapper.tui.files.stats.previously_completed += 1
if referer and url != referer and str(referer) != current_referer:
# Update the referer if it has changed so that check_complete_by_referer can work
@@ -498,7 +492,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
return
if await self.check_skip_by_config(media_item):
- self.manager.progress_manager.download_progress.add_skipped()
+ self.manager.scrape_mapper.tui.files.stats.skipped += 1
return
self.manager.scrape_mapper.create_download_task(self._download(media_item, m3u8))
@@ -535,7 +529,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
downloaded = await self.manager.database.history.check_complete_by_referer(domain, url)
if downloaded:
logger.info(f"Skipping {url} as it has already been downloaded")
- self.manager.progress_manager.download_progress.add_previously_completed()
+ self.manager.scrape_mapper.tui.files.stats.previously_completed += 1
return downloaded
@final
@@ -547,7 +541,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
if downloaded:
url = _url(scrape_item)
logger.info(f"Skipping {url} as its hash ({hash_type}:{hash_value}) has already been downloaded")
- self.manager.progress_manager.download_progress.add_previously_completed()
+ self.manager.scrape_mapper.tui.files.stats.previously_completed += 1
return downloaded
@final
@@ -595,7 +589,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
url_path = self.__db_path__(url)
if album_results.get(url_path) is True:
logger.info(f"Skipping {url} as it has already been downloaded")
- self.manager.progress_manager.download_progress.add_previously_completed()
+ self.manager.scrape_mapper.tui.files.stats.previously_completed += 1
return True
return False
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index 8084bfe1b..9bd9bb35a 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -19,7 +19,6 @@ if TYPE_CHECKING:
from collections.abc import Callable, Generator
import aiohttp
- from rich.progress import TaskID
from cyberdrop_dl import signature
from cyberdrop_dl.managers.manager import Manager
@@ -158,8 +157,6 @@ class MediaItem:
hash: str | None = None
downloaded: bool = field(default=False)
- parent_media_item: MediaItem | None = None
- _task_id: TaskID | None = None
metadata: object = field(init=False, default_factory=dict)
uploaded_at_date: datetime.datetime | None = field(init=False, default=None)
@@ -217,27 +214,9 @@ class MediaItem:
original_filename=original_filename or filename,
parents=origin.parents.copy(),
uploaded_at=origin.uploaded_at,
- parent_media_item=None if isinstance(origin, ScrapeItem) else origin,
parent_threads=origin.parent_threads.copy(),
)
- @property
- def task_id(self) -> TaskID | None:
- if self.parent_media_item is not None:
- return self.parent_media_item.task_id
- return self._task_id
-
- def set_task_id(self, task_id: TaskID | None) -> None:
- if self.task_id is not None and task_id is not None:
- # We already have a task_id; we can't replace it, only reset it.
- # This should never happen. Calling code should always check the value before making a new task.
- # We can't silently ignore it either because we will lose any reference to the created task.
- raise ValueError("task_id is already set")
- if self.parent_media_item is not None:
- self.parent_media_item.set_task_id(task_id)
- else:
- self._task_id = task_id
-
def serialize(self) -> dict[str, Any]:
me = asdict(self)
if self.hash:
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index e554c7208..efe9d3ebf 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -81,7 +81,6 @@ class Downloader:
if not e.retry:
raise
- self.attempt_task_removal(media_item)
if e.status != 999:
media_item.attempts += 1
@@ -108,11 +107,6 @@ class Downloader:
if self.manager.config.settings.sorting.sort_downloads:
self.manager.config.settings.sorting.sort_folder.mkdir(parents=True, exist_ok=True)
- def update_queued_files(self, increase_total: bool = True):
- queued_files = self.manager.progress_manager.file_progress.get_queue_length()
- self.manager.progress_manager.download_progress.update_queued(queued_files)
- self.manager.progress_manager.download_progress.update_total(increase_total)
-
@contextlib.asynccontextmanager
async def _download_context(self, media_item: MediaItem):
@@ -123,7 +117,6 @@ class Downloader:
return
self.waiting_items += 1
- self.update_queued_files()
server = (media_item.debrid_link or media_item.url).host
server_limit, domain_limit, global_limit = (
@@ -134,7 +127,6 @@ class Downloader:
async with server_limit, domain_limit, global_limit:
self.processed_items.add(media_item.db_path)
- self.update_queued_files(increase_total=False)
self.waiting_items -= 1
yield
@@ -167,23 +159,27 @@ class Downloader:
# TODO: compute approx size for UI from the m3u8 info
media_item.download_filename = media_item.path.name
await self.manager.database.history.add_download_filename(self.domain, media_item)
- task_id = self.manager.progress_manager.file_progress.add_task(domain=self.domain, filename=media_item.filename)
- media_item.set_task_id(task_id)
- video, audio, _subs = await self._download_rendition_group(media_item, m3u8_group)
- if not audio:
- await aio.move(video, media_item.path)
- else:
- # TODO: add remux method to ffmpeg to create an mkv file instead of mp4
- # Subtitles format may be incompatible with mp4 and they will be silently dropped by ffmpeg
- # so we leave them as independent files for now
- ffmpeg_result = await ffmpeg.merge((video, audio), media_item.path)
-
- if not ffmpeg_result.success:
- raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)
-
- await self.client.process_completed(media_item, self.domain)
- await self.client.handle_media_item_completion(media_item, downloaded=True)
- await self.finalize_download(media_item, downloaded=True)
+
+ with self.manager.scrape_mapper.tui.downloads.download_hls(
+ media_item.filename,
+ media_item.domain,
+ segments=sum(len(m.segments) for m in m3u8_group if m is not None),
+ ):
+ video, audio, _subs = await self._download_rendition_group(media_item, m3u8_group)
+ if not audio:
+ await aio.move(video, media_item.path)
+ else:
+ # TODO: add remux method to ffmpeg to create an mkv file instead of mp4
+ # Subtitles format may be incompatible with mp4 and they will be silently dropped by ffmpeg
+ # so we leave them as independent files for now
+ ffmpeg_result = await ffmpeg.merge((video, audio), media_item.path)
+
+ if not ffmpeg_result.success:
+ raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)
+
+ await self.client.process_completed(media_item, self.domain)
+ await self.client.handle_media_item_completion(media_item, downloaded=True)
+ await self.finalize_download(media_item, downloaded=True)
async def _download_rendition_group(
self, media_item: MediaItem, m3u8_group: Rendition
@@ -283,8 +279,7 @@ class Downloader:
if downloaded:
await aio.chmod(media_item.path, 0o666)
await self.set_file_datetime(media_item, media_item.path)
- self.attempt_task_removal(media_item)
- self.manager.progress_manager.download_progress.add_completed()
+ self.manager.scrape_mapper.tui.files.stats.completed += 1
logger.info(f"Download finished: {media_item.url}")
"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
@@ -320,18 +315,6 @@ class Downloader:
except OSError:
pass
- def attempt_task_removal(self, media_item: MediaItem) -> None:
- """Attempts to remove the task from the progress bar."""
- if media_item.is_segment:
- return
- if media_item.task_id is not None:
- try:
- self.manager.progress_manager.file_progress.remove_task(media_item.task_id)
- except ValueError:
- pass
-
- media_item.set_task_id(None)
-
"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
async def start_download(self, media_item: MediaItem) -> bool:
@@ -366,16 +349,14 @@ class Downloader:
await aio.chmod(media_item.path, 0o666)
if not media_item.is_segment:
await self.set_file_datetime(media_item, media_item.path)
- self.manager.progress_manager.download_progress.add_completed()
+ self.manager.scrape_mapper.tui.files.stats.completed += 1
logger.info(f"Download finished: {media_item.url}")
- self.attempt_task_removal(media_item)
return downloaded
except SkipDownloadError as e:
if not media_item.is_segment:
logger.info(f"Download skipped {media_item.url}: {e}")
- self.manager.progress_manager.download_progress.add_skipped()
- self.attempt_task_removal(media_item)
+ self.manager.scrape_mapper.tui.files.stats.skipped += 1
except (DownloadError, ClientResponseError, InvalidContentTypeError):
raise
@@ -405,11 +386,10 @@ class Downloader:
error_log_msg: ErrorLogMessage,
exc_info: Exception | None = None,
) -> None:
- self.attempt_task_removal(media_item)
logger.error(
f"{self.log_prefix} Failed: {media_item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {media_item.referer}",
exc_info=exc_info,
)
self.manager.logs.write_download_error(media_item, error_log_msg.csv_log_msg)
- self.manager.progress_manager.download_stats_progress.add_failure(error_log_msg.ui_failure)
- self.manager.progress_manager.download_progress.add_failed()
+ self.manager.scrape_mapper.tui.files.stats.failed += 1
+ self.manager.scrape_mapper.tui.scrape_errors.add(error_log_msg.ui_failure)
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index 39d4bcae4..b552ca4ab 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import asyncio
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, final
import aiofiles
from mega.chunker import MegaChunker, get_chunks
@@ -17,20 +17,21 @@ if TYPE_CHECKING:
from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.manager import Manager
+ from cyberdrop_dl.progress import ProgressHook
+@final
class MegaDownloadClient(DownloadClient):
def __init__(self, manager: Manager) -> None:
super().__init__(manager, manager.client_manager)
self._decrypt_mapping: dict[URL, tuple[Crypto, int]] = {}
self._supports_ranges = False
- async def _append_content(self, media_item: MediaItem, content: aiohttp.StreamReader) -> None:
+ async def _append_content(self, media_item: MediaItem, hook: ProgressHook, content: aiohttp.StreamReader) -> None:
"""Appends content to a file."""
- assert media_item.task_id is not None
check_free_space = storage.create_free_space_checker(media_item)
- check_download_speed = self.make_speed_checker(media_item)
+ check_download_speed = self.make_speed_checker(media_item, hook)
await check_free_space()
await self._pre_download_check(media_item)
@@ -46,7 +47,7 @@ class MegaDownloadClient(DownloadClient):
await self.client_manager.speed_limiter.acquire(chunk_size)
await f.write(chunk)
- self.manager.progress_manager.file_progress.advance_file(media_item.task_id, chunk_size)
+ hook.advance(chunk_size)
check_download_speed()
await self._post_download_check(media_item)
diff --git a/cyberdrop_dl/hasher.py b/cyberdrop_dl/hasher.py
index c8ca560b2..673bcad31 100644
--- a/cyberdrop_dl/hasher.py
+++ b/cyberdrop_dl/hasher.py
@@ -51,7 +51,7 @@ async def hash_directory_scanner(manager: Manager, path: Path) -> None:
async with manager.database:
stats = await manager.hasher.hash_directory(path)
- manager.progress_manager.print_hashing_stats(stats)
+ manager.print_hashing_stats(stats)
@dataclasses.dataclass(slots=True)
@@ -68,7 +68,7 @@ class Hasher:
_dedupe_tui: DedupeUI = dataclasses.field(init=False)
def __post_init__(self) -> None:
- base_dir = self.manager.config.settings.files.download_folder
+ base_dir = self.manager.config.settings.files.download_folder.expanduser().resolve().absolute()
self._tui = HashingUI(base_dir)
self._dedupe_tui = DedupeUI(base_dir)
diff --git a/cyberdrop_dl/managers/live_manager.py b/cyberdrop_dl/managers/live_manager.py
deleted file mode 100644
index a0fa7a25c..000000000
--- a/cyberdrop_dl/managers/live_manager.py
+++ /dev/null
@@ -1,104 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import shutil
-from contextlib import contextmanager
-from functools import partialmethod
-from typing import TYPE_CHECKING
-
-from rich.live import Live
-
-from cyberdrop_dl import env
-from cyberdrop_dl.logs import LOG_TO_CONSOLE
-
-if TYPE_CHECKING:
- from collections.abc import Generator
-
- from rich.console import RenderableType
-
- from cyberdrop_dl.managers.manager import Manager
-
-
-def is_terminal_in_portrait() -> bool:
- """Check if CDL is being run in portrait mode based on a few conditions."""
-
- if env.PORTRAIT_MODE:
- return True
-
- terminal_size = shutil.get_terminal_size()
- width, height = terminal_size.columns, terminal_size.lines
- aspect_ratio = width / height
-
- # High aspect ratios are likely to be in landscape mode
- if aspect_ratio >= 3.2:
- return False
-
- # Check for mobile device in portrait mode
- if (aspect_ratio < 1.5 and height >= 40) or (width <= 85 and aspect_ratio < 2.3):
- return True
-
- # Assume landscape mode for other cases
- return False
-
-
-class LiveManager:
- def __init__(self, manager: Manager) -> None:
- self.manager = manager
- self.ui_setting = self.manager.cli_args.ui
- self.fullscreen = f = self.manager.cli_args.fullscreen_ui
- self.refresh_rate = rate = self.manager.config.global_settings.ui_options.refresh_rate
- self.live = Live(refresh_per_second=rate, transient=True, screen=f, auto_refresh=True)
- self.current_layout: str = ""
-
- @contextmanager
- def get_live(self, name: str, stop: bool = False) -> Generator[Live | None]:
- layout = self.get_layout(name)
- token = LOG_TO_CONSOLE.set(False)
- try:
- with self.live_context_manager(layout, stop=stop) as live:
- yield live
- finally:
- LOG_TO_CONSOLE.reset(token)
-
- get_sort_live = partialmethod(get_live, name="sort_layout")
- get_main_live = partialmethod(get_live, name="main_layout")
- get_hash_live = partialmethod(get_live, name="hash_layout")
- get_remove_file_via_hash_live = partialmethod(get_live, name="hash_remove_layout")
-
- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
-
- def get_layout(self, name: str) -> RenderableType | None:
- if name == "main_layout":
- name = f"{self.ui_setting.value}_layout"
- self.current_layout = name
- return getattr(self.manager.progress_manager, name, None)
-
- async def watch_orientation(self, stop_event: asyncio.Event) -> None:
- """Watches for screen orientation changes and updates the live display accordingly."""
- while not stop_event.is_set():
- new_layout = "vertical_layout" if is_terminal_in_portrait() else "horizontal_layout"
- if new_layout != self.current_layout:
- self.current_layout = new_layout
- layout = self.get_layout(new_layout)
- self.live.update(layout, refresh=True) # type: ignore[reportArgumentType]
- await asyncio.sleep(0.5)
-
- @contextmanager
- def live_context_manager(self, layout: RenderableType | None, stop: bool = False) -> Generator[Live | None]:
- stop_event = asyncio.Event()
- orientation_task = None
-
- try:
- self.live.start()
- if layout:
- self.live.update(layout, refresh=True)
- if self.current_layout in ("vertical_layout", "horizontal_layout"):
- orientation_task = asyncio.create_task(self.watch_orientation(stop_event))
- yield self.live
- finally:
- stop_event.set()
- if orientation_task:
- orientation_task.cancel()
- if stop:
- self.live.update("")
- self.live.stop()
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index a3378c798..81486bb1f 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -2,25 +2,33 @@ from __future__ import annotations
import dataclasses
import logging
+import time
+from datetime import timedelta
from pathlib import Path
from typing import TYPE_CHECKING, Any, Self
+from pydantic.types import ByteSize
+
from cyberdrop_dl import __version__, ffmpeg, yaml
from cyberdrop_dl.cli import CLIargs
from cyberdrop_dl.config import Config
from cyberdrop_dl.database import Database
from cyberdrop_dl.hasher import Hasher
+from cyberdrop_dl.logs import capture_logs, log_spacer
from cyberdrop_dl.managers.client_manager import ClientManager
-from cyberdrop_dl.managers.live_manager import LiveManager
from cyberdrop_dl.managers.logs import LogManager
-from cyberdrop_dl.managers.progress_manager import ProgressManager
from cyberdrop_dl.utils.utilities import get_system_information
if TYPE_CHECKING:
+ from collections.abc import Sequence
from os import PathLike
from cyberdrop_dl.data_structures.url_objects import MediaItem
- from cyberdrop_dl.scrape_mapper import ScrapeMapper
+ from cyberdrop_dl.progress.dedupe import DedupeStats
+ from cyberdrop_dl.progress.hashing import HashingStats
+ from cyberdrop_dl.progress.scraping.errors import Error
+ from cyberdrop_dl.progress.sorting import SortStats
+ from cyberdrop_dl.scrape_mapper import ScrapeMapper, ScrapeStats
logger = logging.getLogger(__name__)
@@ -42,9 +50,6 @@ class Manager:
self._completed_downloads: list[MediaItem] = []
self.hasher: Hasher = Hasher(self)
self.logs: LogManager = LogManager.from_manager(self)
- self.live_manager: LiveManager = LiveManager(self)
- self.progress_manager: ProgressManager = ProgressManager(self)
-
self.scrape_mapper: ScrapeMapper
self.database: Database
self.client_manager: ClientManager
@@ -115,6 +120,107 @@ class Manager:
async def close(self) -> None:
await self.client_manager.close()
+ def print_stats(self, stats: ScrapeStats) -> str:
+ if not self.cli_args.print_stats:
+ return ""
+
+ log_spacer()
+
+ with capture_logs() as stream:
+ self._print_stats(stats)
+
+ return stream.getvalue()
+
+ def _print_stats(self, stats: ScrapeStats) -> None:
+
+ elapsed = timedelta(seconds=int(time.monotonic() - stats.start_time))
+ total_data_written = ByteSize(self.scrape_mapper.tui.downloads.bytes_downloaded).human_readable(decimal=True)
+
+ logger.info("Run Stats:", extra={"color": "cyan"})
+ logger.info(f" Config file: {self.config.source}")
+ logger.info(f" URLs source: {stats.source}")
+ logger.info(f" URLs: {stats.count:,}")
+ logger.info(f" URL groups: {len(stats.unique_groups):,}")
+ logger.info(f" Logs folder: {self.config.settings.logs.log_folder}")
+ logger.info(f" Total runtime: {elapsed}")
+ logger.info(f" Total downloaded data: {total_data_written}")
+
+ if stats.domain_stats:
+ log_spacer()
+ logger.info("URLs by domain:", extra={"color": "cyan"})
+ logger.info(" - " + "\n - ".join(map(str, stats.domain_stats.items())))
+
+ log_spacer()
+ logger.info("Download Stats:", extra={"color": "cyan"})
+ logger.info(f" Downloaded: {self.scrape_mapper.tui.files.stats.completed:,} files")
+ logger.info(f" Skipped (by config): {self.scrape_mapper.tui.files.stats.skipped:,} files")
+ logger.info(
+ f" Skipped (previously downloaded): {self.scrape_mapper.tui.files.stats.previously_completed:,} files"
+ )
+ logger.info(f" Failed: {self.scrape_mapper.tui.files.stats.failed:,} files")
+
+ log_spacer()
+ logger.info("Unsupported URLs Stats:", extra={"color": "cyan"})
+ logger.info(f" Sent to Jdownloader: {self.scrape_mapper.tui.scrape_errors.sent_to_jdownloader:,}")
+ logger.info(f" Skipped: {self.scrape_mapper.tui.scrape_errors.skipped:,}")
+
+ hash_stats, dedupe_stats = self.hasher.stats
+ self.print_hashing_stats(hash_stats)
+ self.print_dedupe_stats(dedupe_stats)
+ # self.print_sort_stats()
+ self.print_errors()
+
+ def print_sort_stats(self, stats: SortStats):
+ log_spacer()
+ logger.info("Sort Stats:", extra={"color": "cyan"})
+ logger.info(f" Audios: {stats.audios:,}")
+ logger.info(f" Images: {stats.images:,}")
+ logger.info(f" Videos: {stats.videos:,}")
+ logger.info(f" Other Files: {stats.others:,}")
+
+ def print_errors(self):
+ _log_errors(
+ tuple(self.scrape_mapper.tui.scrape_errors),
+ tuple(self.scrape_mapper.tui.download_errors),
+ )
+
+ def print_hashing_stats(self, stats: HashingStats) -> None:
+ log_spacer()
+ logger.info("Checksum Stats:", extra={"color": "cyan"})
+ logger.info(f" Newly hashed: {stats.new_hashed:,} files")
+ logger.info(f" Previously hashed: {stats.prev_hashed:,} files")
+
+ def print_dedupe_stats(self, stats: DedupeStats) -> None:
+ log_spacer()
+ logger.info("Dupe Stats:", extra={"color": "cyan"})
+ logger.info(f" Deleted (duplicates of previous downloads): {stats.deleted:,} files")
+ logger.info(f" Errors: {stats.total - stats.deleted:,} files")
+
+
+def _log_errors(scrape_errors: Sequence[Error], download_errors: Sequence[Error]) -> None:
+ error_codes = (error.code for error in (*scrape_errors, *download_errors) if error.code is not None)
+
+ try:
+ padding = len(str(max(error_codes)))
+ except ValueError:
+ padding = 0
+
+ for title, errors in (
+ ("Scrape Failures:", scrape_errors),
+ ("Download Failures:", download_errors),
+ ):
+ log_spacer()
+ logger.info(title, extra={"color": "cyan"})
+ if not errors:
+ logger.info(f" {'None':>{padding}}", extra={"color": "green"})
+ continue
+
+ for error in errors:
+ error_code = error.code if error.code is not None else ""
+ logger.info(
+ f" {error_code:>{padding}}{' ' if padding else ''}{error.msg}: {error.count:,}", extra={"color": "red"}
+ )
+
@dataclasses.dataclass(slots=True, frozen=True, kw_only=True)
class AppData:
diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.py
index afe492228..19b253623 100644
--- a/cyberdrop_dl/managers/progress_manager.py
+++ b/cyberdrop_dl/managers/progress_manager.py
@@ -1,211 +1,10 @@
from __future__ import annotations
import logging
-import time
-from contextlib import asynccontextmanager
-from datetime import timedelta
-from typing import TYPE_CHECKING
-
-from pydantic import ByteSize
-from rich.columns import Columns
-from rich.console import Group
-from rich.layout import Layout
-from rich.progress import Progress, SpinnerColumn, TaskID
-
-from cyberdrop_dl import __version__
-from cyberdrop_dl.logs import capture_logs, log_spacer
-from cyberdrop_dl.ui.progress.downloads_progress import DownloadsProgress
-from cyberdrop_dl.ui.progress.file_progress import FileProgress
-from cyberdrop_dl.ui.progress.hash_progress import HashProgress
-from cyberdrop_dl.ui.progress.scraping_progress import ScrapingProgress
-from cyberdrop_dl.ui.progress.sort_progress import SortProgress
-from cyberdrop_dl.ui.progress.statistic_progress import DownloadStatsProgress, ScrapeStatsProgress
-
-if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Sequence
-
- from rich.panel import Panel
-
- from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.progress.dedupe import DedupeStats
- from cyberdrop_dl.progress.hashing import HashingStats
- from cyberdrop_dl.scrape_mapper import ScrapeStats
- from cyberdrop_dl.ui.progress.statistic_progress import UiFailureTotal
+from rich.progress import SpinnerColumn
logger = logging.getLogger(__name__)
spinner = SpinnerColumn(style="green", spinner_name="dots")
-
-
-class ProgressManager:
- def __init__(self, manager: Manager) -> None:
- # File Download Bars
- self.manager = manager
- ui_options = manager.config.global_settings.ui_options
- self.portrait = manager.cli_args.portrait
- self.file_progress = FileProgress(manager)
- self.scraping_progress = ScrapingProgress(manager)
-
- # Overall Progress Bars & Stats
- self.download_progress: DownloadsProgress = DownloadsProgress(manager)
- self.download_stats_progress: DownloadStatsProgress = DownloadStatsProgress()
- self.scrape_stats_progress: ScrapeStatsProgress = ScrapeStatsProgress()
- self.hash_progress: HashProgress = HashProgress(manager)
- self.sort_progress: SortProgress = SortProgress(1, manager)
-
- self.ui_refresh_rate: int = ui_options.refresh_rate
-
- activity = Progress(spinner, "[progress.description]{task.description}")
- self.status_message: Progress = Progress(spinner, "[progress.description]{task.description}")
-
- self.status_message_task_id: TaskID = self.status_message.add_task("", total=100, completed=0, visible=False)
- self.activity_task_id: TaskID = activity.add_task(
- f"Running Cyberdrop-DL: v{__version__}", total=100, completed=0
- )
- self.activity: Progress = activity
-
- simple_layout = Group(activity, self.download_progress.simple_progress)
-
- status_message_columns = Columns([activity, self.status_message], expand=False)
-
- horizontal_layout = Layout()
- vertical_layout = Layout()
-
- upper_layouts = (
- Layout(renderable=self.download_progress.get_progress(), name="Files", ratio=1, minimum_size=9),
- Layout(renderable=self.scrape_stats_progress.get_progress(), name="Scrape Failures", ratio=1),
- Layout(renderable=self.download_stats_progress.get_progress(), name="Download Failures", ratio=1),
- )
-
- lower_layouts = (
- Layout(renderable=self.scraping_progress.get_renderable(), name="Scraping", ratio=20),
- Layout(renderable=self.file_progress.get_renderable(), name="Downloads", ratio=20),
- Layout(renderable=status_message_columns, name="status_message", ratio=2),
- )
-
- horizontal_layout.split_column(Layout(name="upper", ratio=20), *lower_layouts)
- vertical_layout.split_column(Layout(name="upper", ratio=60), *lower_layouts)
-
- horizontal_layout["upper"].split_row(*upper_layouts)
- vertical_layout["upper"].split_column(*upper_layouts)
-
- self.horizontal_layout: Layout = horizontal_layout
- self.vertical_layout: Layout = vertical_layout
- self.activity_layout: Progress = activity
- self.simple_layout: Group = simple_layout
- self.hash_remove_layout: Panel = self.hash_progress.get_removed_progress()
- self.hash_layout: Panel = self.hash_progress.get_renderable()
- self.sort_layout: Panel = self.sort_progress.get_renderable()
-
- @asynccontextmanager
- async def show_status_msg(self, msg: str | None) -> AsyncGenerator[None]:
- try:
- self.status_message.update(self.status_message_task_id, description=msg, visible=bool(msg))
- yield
- finally:
- self.status_message.update(self.status_message_task_id, visible=False)
-
- @property
- def fullscreen_layout(self) -> Layout:
- if self.portrait:
- return self.vertical_layout
- return self.horizontal_layout
-
- def print_stats(self, stats: ScrapeStats) -> str:
- if not self.manager.cli_args.print_stats:
- return ""
-
- log_spacer()
- logger.info("Printing Stats...\n")
-
- with capture_logs() as stream:
- self._print_stats(stats)
-
- return stream.getvalue()
-
- def _print_stats(self, stats: ScrapeStats) -> None:
-
- elapsed = timedelta(seconds=int(time.monotonic() - stats.start_time))
- total_data_written = ByteSize(self.file_progress.total_data_written).human_readable(decimal=True)
-
- logger.info("Run Stats:", extra={"color": "cyan"})
- logger.info(f" Config file: {self.manager.config.source}")
- logger.info(f" URLs source: {stats.source}")
- logger.info(f" URLs: {stats.count:,}")
- logger.info(f" URL groups: {len(stats.unique_groups):,}")
- logger.info(f" Logs folder: {self.manager.config.settings.logs.log_folder}")
- logger.info(f" Total runtime: {elapsed}")
- logger.info(f" Total downloaded data: {total_data_written}")
-
- if stats.domain_stats:
- log_spacer()
- logger.info("URLs by domain:", extra={"color": "cyan"})
- logger.info(" - " + "\n - ".join(map(str, stats.domain_stats.items())))
-
- log_spacer()
- logger.info("Download Stats:", extra={"color": "cyan"})
- logger.info(f" Downloaded: {self.download_progress.completed_files:,} files")
- logger.info(f" Skipped (by config): {self.download_progress.skipped_files:,} files")
- logger.info(f" Skipped (previously downloaded): {self.download_progress.previously_completed_files:,} files")
- logger.info(f" Failed: {self.download_stats_progress.failed_files:,} files")
-
- log_spacer()
- logger.info("Unsupported URLs Stats:", extra={"color": "cyan"})
- logger.info(f" Sent to Jdownloader: {self.scrape_stats_progress.sent_to_jdownloader:,}")
- logger.info(f" Skipped: {self.scrape_stats_progress.unsupported_urls_skipped:,}")
-
- hash_stats, dedupe_stats = self.manager.hasher.stats
- self.print_hashing_stats(hash_stats)
- self.print_dedupe_stats(dedupe_stats)
-
- log_spacer()
- logger.info("Sort Stats:", extra={"color": "cyan"})
- logger.info(f" Audios: {self.sort_progress.audio_count:,}")
- logger.info(f" Images: {self.sort_progress.image_count:,}")
- logger.info(f" Videos: {self.sort_progress.video_count:,}")
- logger.info(f" Other Files: {self.sort_progress.other_count:,}")
-
- self.print_errors()
-
- def print_errors(self):
-
- _log_errors(self.scrape_stats_progress.return_totals(), self.download_stats_progress.return_totals())
-
- def print_hashing_stats(self, stats: HashingStats) -> None:
- log_spacer()
- logger.info("Checksum Stats:", extra={"color": "cyan"})
- logger.info(f" Newly hashed: {stats.new_hashed:,} files")
- logger.info(f" Previously hashed: {stats.prev_hashed:,} files")
-
- def print_dedupe_stats(self, stats: DedupeStats) -> None:
- log_spacer()
- logger.info("Dupe Stats:", extra={"color": "cyan"})
- logger.info(f" Deleted (duplicates of previous downloads): {stats.deleted:,} files")
- logger.info(f" Errors: {stats.total - stats.deleted:,} files")
-
-
-def _log_errors(scrape_errors: Sequence[UiFailureTotal], download_errors: Sequence[UiFailureTotal]) -> None:
- error_codes = (error.code for error in (*scrape_errors, *download_errors) if error.code is not None)
-
- try:
- padding = len(str(max(error_codes)))
- except ValueError:
- padding = 0
-
- for title, errors in (
- ("Scrape Failures:", scrape_errors),
- ("Download Failures:", download_errors),
- ):
- log_spacer()
- logger.info(title, extra={"color": "cyan"})
- if not errors:
- logger.info(f" {'None':>{padding}}", extra={"color": "green"})
- continue
-
- for error in errors:
- error_code = error.code if error.code is not None else ""
- logger.info(
- f" {error_code:>{padding}}{' ' if padding else ''}{error.msg}: {error.total:,}", extra={"color": "red"}
- )
diff --git a/cyberdrop_dl/ui/program_ui.py b/cyberdrop_dl/program_ui.py
similarity index 100%
rename from cyberdrop_dl/ui/program_ui.py
rename to cyberdrop_dl/program_ui.py
diff --git a/cyberdrop_dl/progress/__init__.py b/cyberdrop_dl/progress/__init__.py
index b62629ca3..ef0bfa502 100644
--- a/cyberdrop_dl/progress/__init__.py
+++ b/cyberdrop_dl/progress/__init__.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import contextlib
import dataclasses
+import sys
from abc import ABC, abstractmethod
from contextvars import ContextVar
from typing import TYPE_CHECKING, Self
@@ -92,7 +93,7 @@ class LiveUI(ABC):
def __rich__(self) -> RenderableType: ...
@contextlib.contextmanager
- def __call__(self, *, transient: bool = False) -> Generator[None]:
+ def __call__(self, *, transient: bool = True) -> Generator[None]:
if self.disabled:
yield None
return
@@ -101,9 +102,10 @@ class LiveUI(ABC):
try:
with Live(
refresh_per_second=REFRESH_RATE.get(),
- auto_refresh=True,
+ auto_refresh="pytest" not in sys.modules,
+ screen=transient,
transient=transient,
- get_renderable=self.__rich__,
+ get_renderable=self.__rich__ if "pytest" not in sys.modules else None,
):
yield
finally:
diff --git a/cyberdrop_dl/progress/overflow.py b/cyberdrop_dl/progress/overflow.py
index fea91faa5..931bb35c0 100644
--- a/cyberdrop_dl/progress/overflow.py
+++ b/cyberdrop_dl/progress/overflow.py
@@ -12,6 +12,8 @@ from rich.progress import Task, TaskID
from cyberdrop_dl.progress import DictProgress
if TYPE_CHECKING:
+ from collections.abc import Callable
+
from rich.progress import ProgressColumn, Task, TaskID
_COLOR: str = "plum3"
@@ -54,12 +56,19 @@ class OverFlow:
class OverFlowPanel:
unit: ClassVar[str]
- def __init__(self, *columns: ProgressColumn | str, max_rows: int, expand: bool = True) -> None:
+ def __init__(
+ self,
+ *columns: ProgressColumn | str,
+ max_rows: int,
+ expand: bool = True,
+ get_queue: Callable[[], int] = lambda: 0,
+ ) -> None:
self.max_rows: int = max_rows
self._progress: Final[DictProgress] = DictProgress(*columns, expand=expand)
self._overflow: Final[OverFlow] = OverFlow(self.unit)
self._invisible_rows: Final[deque[TaskID]] = deque()
self._visible_rows: int = 0
+ self.get_queue: Callable[[], int] = get_queue
self._panel: Final[Panel] = Panel(
Group(self._progress, self._overflow),
title=type(self).__name__.removesuffix("Panel"),
@@ -70,6 +79,7 @@ class OverFlowPanel:
@final
def __rich__(self) -> Panel:
self._overflow.count = len(self._progress) - self._visible_rows
+ self._overflow.queued = self.get_queue()
return self._panel
@final
diff --git a/cyberdrop_dl/progress/scraping/downloads.py b/cyberdrop_dl/progress/scraping/downloads.py
index b9659c934..7a585ce72 100644
--- a/cyberdrop_dl/progress/scraping/downloads.py
+++ b/cyberdrop_dl/progress/scraping/downloads.py
@@ -10,6 +10,7 @@ from pathlib import Path
from typing import TYPE_CHECKING, ClassVar, Final, final
from rich.jupyter import JupyterMixin
+from rich.markup import escape
from rich.measure import Measurement
from rich.progress import (
BarColumn,
@@ -140,7 +141,13 @@ class DownloadsPanel(OverFlowPanel):
self._total_bytes = 0
@contextlib.contextmanager
- def download_hls(self, filename: str, /, segments: float | None = None) -> Generator[None]:
+ def download_hls(
+ self,
+ filename: str,
+ /,
+ domain: str,
+ segments: float,
+ ) -> Generator[None]:
# For HLS downloads, we use 2 different tasks. One on a hidden progress to track the downloaded bytes
# and one on the user facing progress to track the number of downloaded segments (with a known total)
# We create both at the same time and smuggle the bytes task as a field of the segments task
@@ -148,7 +155,8 @@ class DownloadsPanel(OverFlowPanel):
task_id = self._hls_progress.add_task("", total=None, visible=False)
filename = str(filename).rsplit("/", 1)[-1]
- segments_task = self._add_task(filename, segments)
+ desc = escape((f"({domain.upper()}) {filename}").encode().decode("ascii", errors="ignore"))
+ segments_task = self._add_task(desc, segments)
bytes_task = self._hls_progress[task_id]
self._progress.update(segments_task.id, HLS=bytes_task)
token = _current_hls_task.set(segments_task.id)
@@ -159,9 +167,16 @@ class DownloadsPanel(OverFlowPanel):
self._hls_progress.remove_task(task_id)
_current_hls_task.reset(token)
- def download_file(self, description: object, /, total: float | None = None) -> ProgressHook:
+ def download_file(
+ self,
+ description: object,
+ /,
+ domain: str,
+ total: float | None,
+ ) -> ProgressHook:
filename = str(description).rsplit("/", 1)[-1]
- task = self._add_task(filename, total)
+ desc = escape((f"({domain.upper()}) {filename}").encode().decode("ascii", errors="ignore"))
+ task = self._add_task(desc, total)
def advance(amount: int = 1) -> None:
self._total_bytes += amount
@@ -204,7 +219,7 @@ class DownloadsPanel(OverFlowPanel):
async def download_file(filename: str) -> None:
size = random.randint(int(1e2), int(1e9))
- hook = self.download_file(filename, size)
+ hook = self.download_file(filename, "example.com", size)
await download(hook, size)
async def download_hls(filename: str) -> None:
@@ -219,7 +234,7 @@ class DownloadsPanel(OverFlowPanel):
finally:
segments_sem.release()
- with self.download_hls(filename, n_segments):
+ with self.download_hls(filename, "example.com", n_segments):
async with asyncio.TaskGroup() as tg:
for _ in range(n_segments):
await segments_sem.acquire()
diff --git a/cyberdrop_dl/progress/scraping/panel.py b/cyberdrop_dl/progress/scraping/panel.py
index 22c0ffffe..2ffcf62ce 100644
--- a/cyberdrop_dl/progress/scraping/panel.py
+++ b/cyberdrop_dl/progress/scraping/panel.py
@@ -27,7 +27,7 @@ _generate_unique_id = itertools.count(1).__next__
@final
@dataclasses.dataclass(slots=True, frozen=True)
class StatusMessage:
- description: Text | str = f"Running cyberdrop-dl [blue]v{__version__}[/blue]"
+ description: Text | str = f" cyberdrop-dl [blue]v{__version__}[/blue]"
_messages: dict[int, tuple[Spinner, Text]] = dataclasses.field(init=False, default_factory=dict)
_cols: Columns = dataclasses.field(init=False, default_factory=Columns)
@@ -74,7 +74,7 @@ class ScrapingPanel(OverFlowPanel):
@contextlib.contextmanager
def new(self, url: object) -> Generator[None]:
- task = self._add_task(str(url))
+ task = self._add_task(escape(str(url).encode().decode("ascii", errors="ignore")))
try:
yield
finally:
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.py
index 300908879..00e908cbf 100644
--- a/cyberdrop_dl/scrape_mapper.py
+++ b/cyberdrop_dl/scrape_mapper.py
@@ -7,7 +7,6 @@ import datetime
import logging
import re
import time
-from contextvars import ContextVar
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, Self, TypeVar
@@ -25,6 +24,7 @@ from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler, WordPressMedia
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
from cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError
from cyberdrop_dl.logs import log_spacer
+from cyberdrop_dl.progress.scraping import ScrapingUI
from cyberdrop_dl.utils import filepath
from cyberdrop_dl.utils.utilities import get_download_path, remove_trailing_slash
@@ -45,8 +45,6 @@ logger = logging.getLogger(__name__)
REGEX_LINKS = re.compile(r"(?:http.*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]))")
-CURRENT_SCRAPE_DONE: ContextVar[asyncio.Event] = ContextVar("CURRENT_SCRAPE_DONE")
-
def _filter_by_date(scrape_item: ScrapeItem, before: datetime.date | None, after: datetime.date | None) -> bool:
skip = False
@@ -129,12 +127,24 @@ class ScrapeMapper:
_seen_urls: set[AbsoluteHttpURL] = dataclasses.field(init=False, default_factory=set)
_crawlers_disabled_at_runtime: set[str] = dataclasses.field(init=False, default_factory=set)
_factory: CrawlerFactory = dataclasses.field(init=False)
+ tui: ScrapingUI = dataclasses.field(init=False, default_factory=ScrapingUI)
+ _done: asyncio.Event = dataclasses.field(init=False, default_factory=asyncio.Event)
+
+ def _scrape_queue(self) -> int:
+ return sum(f.waiting_items for f in self._factory)
+
+ def _download_queue(self):
+ total = sum(f.downloader.waiting_items for f in self._factory)
+ self.tui.files.stats.queued = total
+ return total
def __post_init__(self) -> None:
self._direct_http = DirectHttpFile(self.manager)
self._jdownloader = JDownloader.from_manager(self.manager)
self._real_debrid = RealDebridCrawler(self.manager)
self._factory = CrawlerFactory(self.manager)
+ self.tui.scrape.get_queue = self._scrape_queue
+ self.tui.downloads.get_queue = self._download_queue
def create_task(self, coro: Coroutine[Any, Any, _T]) -> None:
_ = self._task_groups.scrape.create_task(coro)
@@ -155,30 +165,29 @@ class ScrapeMapper:
@contextlib.asynccontextmanager
async def __call__(self) -> AsyncGenerator[Self]:
+ assert not self._done.is_set()
_ = filepath.MAX_FILE_LEN.set(self.manager.config.global_settings.general.max_file_name_length)
_ = filepath.MAX_FOLDER_LEN.set(self.manager.config.global_settings.general.max_folder_name_length)
await self.manager.client_manager.load_cookie_files()
## IMPORTANT: Order of each context matters!
- async with (
- self.manager.client_manager,
- storage.monitor(self.manager.config.global_settings.general.required_free_space),
- self.manager.logs.task_group,
- self._task_groups.downloads,
- ):
- done = asyncio.Event()
- token = CURRENT_SCRAPE_DONE.set(done)
- try:
- async with self._task_groups.scrape:
- self.manager.scrape_mapper = self
-
- yield self
-
- finally:
- # The done event signals that all scraping is done, but there may still be downloads pending
- done.set()
- CURRENT_SCRAPE_DONE.reset(token)
+ with self.tui():
+ async with (
+ self.manager.client_manager,
+ storage.monitor(self.manager.config.global_settings.general.required_free_space),
+ self.manager.logs.task_group,
+ self._task_groups.downloads,
+ ):
+ try:
+ async with self._task_groups.scrape:
+ self.manager.scrape_mapper = self
+
+ yield self
+
+ finally:
+ # The done event signals that all scraping is done, but there may still be downloads pending
+ self._done.set()
async def run(self) -> ScrapeStats:
self._init_crawlers()
@@ -197,10 +206,10 @@ class ScrapeMapper:
source_name, source = _source(self.manager)
async with contextlib.aclosing(source) as items:
stats = ScrapeStats(source_name)
- done = CURRENT_SCRAPE_DONE.get()
async def wait_until_scrape_is_done() -> None:
- _ = await done.wait()
+ _ = await self._done.wait()
+ self.tui.hide_scrape_panel()
stats.url_count.update(
(crawler.DOMAIN, count) for crawler in self._factory if (count := len(crawler._scraped_items))
)
@@ -266,12 +275,12 @@ class ScrapeMapper:
else:
success = True
- self.manager.progress_manager.scrape_stats_progress.add_unsupported(sent_to_jdownloader=success)
+ self.tui.scrape_errors.add_unsupported(sent_to_jdownloader=success)
return
logger.warning(f"Unsupported URL: {scrape_item.url}")
self.manager.logs.write_unsupported(scrape_item.url, scrape_item.parents[0] if scrape_item.parents else None)
- self.manager.progress_manager.scrape_stats_progress.add_unsupported()
+ self.tui.scrape_errors.add_unsupported()
def _should_scrape(self, scrape_item: ScrapeItem) -> bool:
if scrape_item.url in self._seen_urls:
diff --git a/cyberdrop_dl/ui/progress/__init__.py b/cyberdrop_dl/ui/progress/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/cyberdrop_dl/ui/progress/deque_progress.py b/cyberdrop_dl/ui/progress/deque_progress.py
deleted file mode 100644
index ec1645501..000000000
--- a/cyberdrop_dl/ui/progress/deque_progress.py
+++ /dev/null
@@ -1,118 +0,0 @@
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-from collections import deque
-from itertools import islice
-from typing import TYPE_CHECKING
-
-from rich.console import Group
-from rich.panel import Panel
-from rich.progress import Progress, TaskID
-
-if TYPE_CHECKING:
- from collections.abc import Sequence
-
-
-def adjust_title(s: str, length: int = 40, placeholder: str = "...") -> str:
- """Collapse and truncate or pad the given string to fit in the given length."""
- return f"{s[: length - len(placeholder)]}{placeholder}" if len(s) >= length else s.ljust(length)
-
-
-class DequeProgress(ABC):
- _progress: Progress
- type_str: str = "files"
- color = "plum3"
- progress_str = "[{color}]{description}"
- overflow_str = "[{color}]... and {number:,} other {type_str}"
- queue_str = "[{color}]... and {number:,} {type_str} in {title} queue"
-
- def __init__(self, title: str, visible_tasks_limit: int) -> None:
- self.title = title
- self.title_lower = title.lower()
- self._overflow = Progress("[progress.description]{task.description}")
- self._queue = Progress("[progress.description]{task.description}")
- self._progress_group = Group(self._progress, self._overflow, self._queue)
-
- self._overflow_task_id = self._overflow.add_task(
- self.overflow_str.format(color=self.color, number=0, type_str=self.type_str),
- visible=False,
- )
- self._queue_task_id = self._queue.add_task(
- self.queue_str.format(color=self.color, number=0, type_str=self.type_str, title=self.title_lower),
- visible=False,
- )
- self._tasks: deque[TaskID] = deque()
- self._tasks_visibility_limit = visible_tasks_limit
-
- @abstractmethod
- def get_queue_length(self) -> int: ...
-
- @property
- def visible_tasks(self) -> Sequence[TaskID]:
- if len(self._tasks) > self._tasks_visibility_limit:
- return [self._tasks[i] for i in range(self._tasks_visibility_limit)]
- return self._tasks
-
- @property
- def invisible_tasks(self) -> Sequence[TaskID]:
- return list(islice(self._tasks, self._tasks_visibility_limit, None))
-
- @property
- def invisible_tasks_len(self) -> int:
- """Faster to compute than `len(self.invisible_tasks)`"""
- return max(0, len(self._tasks) - self._tasks_visibility_limit)
-
- def has_visible_capacity(self) -> bool:
- return len(self._tasks) < self._tasks_visibility_limit
-
- def get_renderable(self) -> Panel:
- """Returns the progress bar."""
- return Panel(self._progress_group, title=self.title, border_style="green", padding=(1, 1))
-
- def add_task(self, description: str, total: float | None = None) -> TaskID:
- """Adds a new task to the progress bar."""
- task_id = self._progress.add_task(
- self.progress_str.format(color=self.color, description=description),
- total=total,
- visible=self.has_visible_capacity(),
- )
- self._tasks.append(task_id)
- self.redraw()
- return task_id
-
- def remove_task(self, task_id: TaskID) -> None:
- """Removes a task from the progress bar."""
- if task_id not in self._tasks:
- msg = "Task ID not found"
- raise ValueError(msg)
-
- self._tasks.remove(task_id)
- self._progress.remove_task(task_id)
- self.redraw()
-
- def redraw(self) -> None:
- """Redraws the progress bar."""
- for task in self.visible_tasks:
- self._progress.update(task, visible=True)
-
- invisible_tasks_len = self.invisible_tasks_len
-
- self._overflow.update(
- self._overflow_task_id,
- description=self.overflow_str.format(
- color=self.color,
- number=invisible_tasks_len,
- type_str=self.type_str,
- ),
- visible=invisible_tasks_len > 0,
- )
-
- queue_length = self.get_queue_length()
-
- self._queue.update(
- self._queue_task_id,
- description=self.queue_str.format(
- color=self.color, number=queue_length, type_str=self.type_str, title=self.title_lower
- ),
- visible=queue_length > 0,
- )
diff --git a/cyberdrop_dl/ui/progress/downloads_progress.py b/cyberdrop_dl/ui/progress/downloads_progress.py
deleted file mode 100644
index 3a60017a8..000000000
--- a/cyberdrop_dl/ui/progress/downloads_progress.py
+++ /dev/null
@@ -1,101 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from rich.console import Group
-from rich.panel import Panel
-from rich.progress import BarColumn, Progress
-
-if TYPE_CHECKING:
- from cyberdrop_dl.managers.manager import Manager
-
-
-class DownloadsProgress:
- """Class that keeps track of completed, skipped and failed files."""
-
- def __init__(self, manager: Manager) -> None:
- self.manager = manager
- self.progress = Progress(
- "[progress.description]{task.description}",
- BarColumn(bar_width=None),
- "[progress.percentage]{task.percentage:>6.2f}%",
- "━",
- "{task.completed:,}",
- )
- self.progress_group = Group(self.progress)
-
- self.total_files = 0
- self.completed_files_task_id = self.progress.add_task("[green]Completed", total=0)
- self.completed_files = 0
- self.previously_completed_files_task_id = self.progress.add_task("[yellow]Previously Downloaded", total=0)
- self.previously_completed_files = 0
- self.skipped_files_task_id = self.progress.add_task("[yellow]Skipped By Configuration", total=0)
- self.skipped_files = 0
- self.queued_files_task_id = self.progress.add_task("[cyan]Queued", total=0)
- self.queued_files = 0
- self.failed_files_task_id = self.progress.add_task("[red]Failed", total=0)
- self.failed_files = 0
- self.panel = Panel(
- self.progress_group,
- title="Files",
- border_style="green",
- padding=(1, 1),
- subtitle=f"Total Files: [white]{self.total_files:,}",
- )
- self.simple_progress = Progress(
- "[progress.description]{task.description}",
- BarColumn(bar_width=None),
- "[progress.percentage]{task.percentage:>6.2f}%",
- "━",
- "{task.completed:,}",
- )
- self.simple_progress_task_id = self.simple_progress.add_task("Completed", total=0)
-
- @property
- def simple_completed(self):
- return self.total_files - self.queued_files
-
- def get_progress(self) -> Panel:
- """Returns the progress bar."""
- return self.panel
-
- def update_total(self, increase_total: bool = True) -> None:
- """Updates the total number of files to be downloaded."""
- if increase_total:
- self.total_files = self.total_files + 1
- self.progress.update(self.completed_files_task_id, total=self.total_files)
- self.progress.update(self.previously_completed_files_task_id, total=self.total_files)
- self.progress.update(self.skipped_files_task_id, total=self.total_files)
- self.progress.update(self.failed_files_task_id, total=self.total_files)
- self.progress.update(self.queued_files_task_id, total=self.total_files)
- self.simple_progress.update(
- self.simple_progress_task_id, total=self.total_files, completed=self.simple_completed
- )
- self.panel.subtitle = f"Total Files: [white]{self.total_files:,}"
-
- def add_completed(self) -> None:
- """Adds a completed file to the progress bar."""
- self.progress.advance(self.completed_files_task_id, 1)
- self.completed_files += 1
-
- def add_previously_completed(self, increase_total: bool = True) -> None:
- """Adds a previously completed file to the progress bar."""
- if increase_total:
- self.update_total()
- self.previously_completed_files += 1
- self.progress.advance(self.previously_completed_files_task_id, 1)
-
- def add_skipped(self) -> None:
- """Adds a skipped file to the progress bar."""
- self.progress.advance(self.skipped_files_task_id, 1)
- self.skipped_files += 1
-
- def add_failed(self) -> None:
- """Adds a failed file to the progress bar."""
- self.progress.advance(self.failed_files_task_id, 1)
- self.failed_files += 1
-
- def update_queued(self, number: int) -> None:
- """Adds a queed file to the progress bar."""
- self.queued_files = number
- self.progress.update(self.queued_files_task_id, completed=self.queued_files)
diff --git a/cyberdrop_dl/ui/progress/file_progress.py b/cyberdrop_dl/ui/progress/file_progress.py
deleted file mode 100644
index 13409d468..000000000
--- a/cyberdrop_dl/ui/progress/file_progress.py
+++ /dev/null
@@ -1,79 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from rich.markup import escape
-from rich.progress import (
- BarColumn,
- DownloadColumn,
- Progress,
- SpinnerColumn,
- TaskID,
- TimeRemainingColumn,
- TransferSpeedColumn,
-)
-
-from cyberdrop_dl.ui.progress.deque_progress import DequeProgress, adjust_title
-
-if TYPE_CHECKING:
- from cyberdrop_dl.managers.manager import Manager
-
-
-class FileProgress(DequeProgress):
- """Class that manages the download progress of individual files."""
-
- def __init__(self, manager: Manager) -> None:
- self.manager = manager
- progress_colums = (SpinnerColumn(), "[progress.description]{task.description}", BarColumn(bar_width=None))
- visible_tasks_limit: int = 10
- horizontal_columns = (
- *progress_colums,
- "[progress.percentage]{task.percentage:>6.2f}%",
- "━",
- DownloadColumn(),
- "━",
- TransferSpeedColumn(),
- "━",
- TimeRemainingColumn(),
- )
- vertical_columns = (*progress_colums, DownloadColumn(), "━", TransferSpeedColumn())
- use_columns = horizontal_columns
- if manager.cli_args.portrait:
- use_columns = vertical_columns
- self._progress = Progress(*use_columns)
- self.total_data_written = 0
- super().__init__("Downloads", visible_tasks_limit)
-
- def get_queue_length(self) -> int:
- """Returns the number of tasks in the downloader queue."""
- total = 0
- unique_crawler_ids = set()
- for crawler in self.manager.scrape_mapper._factory:
- crawler_id = id(crawler) # Only count each instance of the crawler once
- if crawler_id in unique_crawler_ids:
- continue
- unique_crawler_ids.add(crawler_id)
- total += getattr(crawler.downloader, "waiting_items", 0)
-
- return total
-
- def add_task(self, *, domain: str, filename: str, expected_size: int | None = None) -> TaskID: # type: ignore[reportIncompatibleMethodOverride]
- """Adds a new task to the progress bar."""
- filename = filename.split("/")[-1].encode("ascii", "ignore").decode().strip()
- description = escape(adjust_title(filename, length=40))
- if not self.manager.progress_manager.portrait:
- description = f"({domain.upper()}) {description}"
- return super().add_task(description, expected_size)
-
- def advance_file(self, task_id: TaskID, amount: int) -> None:
- """Advances the progress of the given task by the given amount."""
- self.total_data_written += amount
- self._progress.advance(task_id, amount)
-
- def get_speed(self, task_id: TaskID) -> float:
- if task_id not in self._tasks:
- msg = "Task ID not found"
- raise ValueError(msg)
-
- task = self._progress._tasks[task_id]
- return task.finished_speed or task.speed or 0
diff --git a/cyberdrop_dl/ui/progress/hash_progress.py b/cyberdrop_dl/ui/progress/hash_progress.py
deleted file mode 100644
index 625044a72..000000000
--- a/cyberdrop_dl/ui/progress/hash_progress.py
+++ /dev/null
@@ -1,125 +0,0 @@
-from __future__ import annotations
-
-import contextlib
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-from pydantic import ByteSize
-from rich.console import Group
-from rich.markup import escape
-from rich.panel import Panel
-from rich.progress import BarColumn, Progress, TaskID
-
-if TYPE_CHECKING:
- from cyberdrop_dl.managers.manager import Manager
-
-
-def _generic_progress() -> Progress:
- return Progress("[progress.description]{task.description}", BarColumn(bar_width=None), "{task.completed:,}")
-
-
-class HashProgress:
- """Class that keeps track of hashed files."""
-
- def __init__(self, manager: Manager) -> None:
- self.manager = manager
- self._hash_progress = _generic_progress()
- self._remove_progress = _generic_progress()
- self._match_progress = _generic_progress()
- self._file_info = Progress("{task.description}")
- self._base_dir: Path | None = None
-
- # hashing
- self._computed_hashes = self._prev_hashed = 0
- self.hash_progress_group = Group(self._file_info, self._hash_progress)
-
- self._tasks: dict[str, TaskID] = {}
-
- def add_hashed_task(hash_type: str) -> None:
- desc = "[green]Hashed " + escape(f"[{hash_type}]")
- self._tasks[hash_type] = self._hash_progress.add_task(desc, total=None)
-
- add_hashed_task("xxh128")
- if manager.config.settings.dupe_cleanup_options.add_md5_hash:
- add_hashed_task("md5")
- if manager.config.settings.dupe_cleanup_options.add_sha256_hash:
- add_hashed_task("sha256")
-
- self.prev_hashed_files_task_id = self._hash_progress.add_task("[green]Previously Hashed", total=None)
-
- self._base_dir_task_id = self._file_info.add_task("")
- self._file_task_id = self._file_info.add_task("")
-
- # remove
- self.removed_files = 0
- self.removed_progress_group = Group(self._match_progress, self._remove_progress)
- self.removed_files_task_id = self._remove_progress.add_task(
- "[green]Removed From Downloaded Files",
- total=None,
- )
-
- @property
- def hashed_files(self) -> int:
- return int(self._computed_hashes / len(self._tasks))
-
- @property
- def prev_hashed_files(self) -> int:
- return int(self._prev_hashed / len(self._tasks))
-
- def get_renderable(self) -> Panel:
- """Returns the progress bar."""
- return Panel(
- self.hash_progress_group,
- title="Hashing",
- border_style="green",
- padding=(1, 1),
- )
-
- def get_removed_progress(self) -> Panel:
- """Returns the progress bar."""
- return Panel(self.removed_progress_group, border_style="green", padding=(1, 1))
-
- @contextlib.contextmanager
- def currently_hashing_dir(self, path: Path):
- self._base_dir = path
- desc = "[green]Base dir: [blue]" + escape(f"{self._base_dir}")
- self._file_info.update(self._base_dir_task_id, description=desc)
- try:
- yield
- finally:
- self._base_dir = None
- self._file_info.update(self._base_dir_task_id, description="")
-
- def update_currently_hashing(self, file: Path) -> None:
- if not self._base_dir:
- return
- file_size = ByteSize(Path(file).stat().st_size)
- size_text = file_size.human_readable(decimal=True)
- path = file.relative_to(self._base_dir)
- desc = "[green]Current file: [blue]" + escape(f"{path}") + f" [green]({size_text})"
- self._file_info.update(self._file_task_id, description=desc)
-
- def add_new_completed_hash(self, hash_type: str) -> None:
- """Adds a completed file to the progress bar."""
- self._hash_progress.advance(self._tasks[hash_type], 1)
- self._computed_hashes += 1
-
- def add_prev_hash(self) -> None:
- """Adds a completed file to the progress bar."""
- self._hash_progress.advance(self.prev_hashed_files_task_id, 1)
- self._prev_hashed += 1
-
- def add_removed_file(self) -> None:
- """Adds a removed file to the progress bar."""
- self._remove_progress.advance(self.removed_files_task_id, 1)
- self.removed_files += 1
-
- def reset(self):
- """Resets the progress bar."""
- for task in self._tasks.values():
- self._hash_progress.reset(task)
- self._hash_progress.reset(self.prev_hashed_files_task_id)
- self._computed_hashes = self._prev_hashed = 0
-
- self._remove_progress.reset(self.removed_files_task_id)
- self.removed_files = 0
diff --git a/cyberdrop_dl/ui/progress/scraping_progress.py b/cyberdrop_dl/ui/progress/scraping_progress.py
deleted file mode 100644
index 8b36e27d4..000000000
--- a/cyberdrop_dl/ui/progress/scraping_progress.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from rich.progress import Progress, SpinnerColumn, TaskID
-
-from cyberdrop_dl.ui.progress.deque_progress import DequeProgress
-
-if TYPE_CHECKING:
- from yarl import URL
-
- from cyberdrop_dl.managers.manager import Manager
-
-
-class ScrapingProgress(DequeProgress):
- """Class that manages the download progress of individual files."""
-
- type_str = "URLs"
-
- def __init__(self, manager: Manager) -> None:
- self.manager = manager
- self._progress = Progress(SpinnerColumn(), "[progress.description]{task.description}")
- visible_tasks_limit: int = 5
- super().__init__("Scraping", visible_tasks_limit)
-
- def get_queue_length(self) -> int:
- """Returns the number of tasks in the scraper queue."""
- total = 0
- unique_crawler_ids = set()
- for crawler in self.manager.scrape_mapper._factory:
- crawler_id = id(crawler) # Only count each instance of the crawler once
- if crawler_id in unique_crawler_ids:
- continue
- unique_crawler_ids.add(crawler_id)
- total += crawler.waiting_items
-
- return total
-
- def redraw(self, passed: bool = False) -> None:
- super().redraw()
- if not passed:
- self.manager.progress_manager.file_progress.redraw()
-
- def add_task(self, url: URL) -> TaskID: # type: ignore[reportIncompatibleMethodOverride]
- """Adds a new task to the progress bar."""
- return super().add_task(str(url))
diff --git a/cyberdrop_dl/ui/progress/sort_progress.py b/cyberdrop_dl/ui/progress/sort_progress.py
deleted file mode 100644
index 5d1aca7f8..000000000
--- a/cyberdrop_dl/ui/progress/sort_progress.py
+++ /dev/null
@@ -1,71 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from rich.markup import escape
-from rich.panel import Panel
-from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID
-
-from cyberdrop_dl.ui.progress.deque_progress import DequeProgress, adjust_title
-
-if TYPE_CHECKING:
- from cyberdrop_dl.managers.manager import Manager
-
-
-class SortProgress(DequeProgress):
- """Class that keeps track of sorted files."""
-
- type_str = "Folders"
-
- def __init__(self, visible_tasks_limit: int, manager: Manager) -> None:
- """Sorter to track the progress of folders being sorted.
-
- Should work similar to the file_progress but for folders, with a percentage and progress bar for the files within the folders"""
- self.manager = manager
- self._progress = Progress(
- SpinnerColumn(),
- "[progress.description]{task.description}",
- BarColumn(bar_width=None),
- "[progress.percentage]{task.percentage:>6.2f}%",
- "━",
- "{task.completed}/{task.total} files",
- )
- super().__init__("Sort", visible_tasks_limit)
-
- # counts
- self.queue_length = self.audio_count = self.video_count = self.image_count = self.other_count = 0
-
- def get_queue_length(self) -> int:
- return self.queue_length
-
- def get_renderable(self) -> Panel:
- """Returns the progress bar."""
- return Panel(
- self._progress_group,
- title="Sorting Downloads",
- border_style="green",
- padding=(1, 1),
- )
-
- def add_task(self, folder: str, expected_size: int | None) -> TaskID:
- """Adds a new task to the progress bar."""
- # description = f'Sorting {folder}'
- description = folder.encode("ascii", "ignore").decode().strip()
- description = escape(adjust_title(description))
- return super().add_task(description, expected_size)
-
- def advance_folder(self, task_id: TaskID, amount: int = 1) -> None:
- """Advances the progress of the given task by the given amount."""
- self._progress.advance(task_id, amount)
-
- def increment_audio(self) -> None:
- self.audio_count += 1
-
- def increment_video(self) -> None:
- self.video_count += 1
-
- def increment_image(self) -> None:
- self.image_count += 1
-
- def increment_other(self) -> None:
- self.other_count += 1
diff --git a/cyberdrop_dl/ui/progress/statistic_progress.py b/cyberdrop_dl/ui/progress/statistic_progress.py
deleted file mode 100644
index b45e3c4ec..000000000
--- a/cyberdrop_dl/ui/progress/statistic_progress.py
+++ /dev/null
@@ -1,186 +0,0 @@
-from __future__ import annotations
-
-import contextlib
-import functools
-from typing import NamedTuple
-
-from rich.console import Group
-from rich.panel import Panel
-from rich.progress import BarColumn, Progress, TaskID
-
-FAILURE_OVERRIDES = {
- "ClientConnectorCertificateError": "Client Connector Certificate Error",
- "ClientConnectorDNSError": "Client Connector DNS Error",
- "ClientConnectorError": "Client Connector Error",
- "ClientConnectorSSLError": "Client Connector SSL Error",
- "ClientHttpProxyError": "Client HTTP Proxy Error",
- "ClientPayloadError": "Client Payload Error",
- "ClientProxyConnectionError": "Client Proxy Connection Error",
- "ConnectionTimeoutError": "Connection Timeout",
- "ContentTypeError": "Content Type Error",
- "InvalidURL": "Invalid URL",
- "InvalidUrlClientError": "Invalid URL Client Error",
- "InvalidUrlRedirectClientError": "Invalid URL Redirect",
- "NonHttpUrlRedirectClientError": "Non HTTP URL Redirect",
- "RedirectClientError": "Redirect Error",
- "ServerConnectionError": "Server Connection Error",
- "ServerDisconnectedError": "Server Disconnected",
- "ServerFingerprintMismatch": "Server Fingerprint Mismatch",
- "ServerTimeoutError": "Server Timeout Error",
- "SocketTimeoutError": "Socket Timeout Error",
-}
-
-
-class TaskInfo(NamedTuple):
- id: TaskID
- description: str
- completed: float
- total: float | None
- progress: float
-
-
-class UiFailureTotal(NamedTuple):
- full_msg: str
- total: int
- code: int | None
- msg: str
-
- @classmethod
- def from_pair(cls, full_msg: str, total: int) -> UiFailureTotal:
- parts = full_msg.split(" ", 1)
- if len(parts) > 1 and parts[0].isdigit():
- error_code, msg = parts
- return cls(full_msg, total, int(error_code), msg)
- return cls(full_msg, total, None, full_msg)
-
-
-def get_tasks_info_sorted(progress: Progress) -> tuple[list[TaskInfo], bool]:
- tasks = [
- TaskInfo(
- id=task.id,
- description=task.description,
- completed=task.completed,
- total=task.total,
- progress=(task.completed / task.total if task.total else 0),
- )
- for task in progress.tasks
- ]
-
- tasks_sorted = sorted(tasks, key=lambda x: x.completed, reverse=True)
- were_sorted = tasks == tasks_sorted
- return tasks_sorted, were_sorted
-
-
-class StatsProgress:
- """Base class that keeps track of failures and reasons."""
-
- title = "Download Failures"
-
- def __init__(self) -> None:
- self.progress = Progress(
- "[progress.description]{task.description}",
- BarColumn(bar_width=None),
- "[progress.percentage]{task.percentage:>6.2f}%",
- "━",
- "{task.completed:,}",
- )
- self.progress_group = Group(self.progress)
- self.failure_types: dict[str, TaskID] = {}
- self.failed_files = 0
- self.panel = Panel(
- self.progress_group,
- title=self.title,
- border_style="green",
- padding=(1, 1),
- subtitle=self.subtitle,
- )
-
- @property
- def subtitle(self) -> str:
- return f"Total {self.title}: [white]{self.failed_files:,}"
-
- def get_progress(self) -> Panel:
- """Returns the progress bar."""
- return self.panel
-
- def update_total(self, total: int) -> None:
- """Updates the total number download failures."""
- self.panel.subtitle = self.subtitle
- for key in self.failure_types:
- self.progress.update(self.failure_types[key], total=total)
-
- tasks_sorted, were_sorted = get_tasks_info_sorted(self.progress)
- if not were_sorted:
- self.sort_tasks(tasks_sorted)
-
- def sort_tasks(self, tasks_sorted: list[TaskInfo]) -> None:
- for task_id in [task.id for task in tasks_sorted]:
- self.progress.remove_task(task_id)
-
- for task in tasks_sorted:
- self.failure_types[task.description] = self.progress.add_task(
- task.description,
- total=task.total,
- completed=task.completed, # type: ignore
- )
-
- def add_failure(self, failure: str) -> None:
- """Adds a failed file to the progress bar."""
- self.failed_files += 1
- key = get_pretty_failure(failure)
- task_id = self.failure_types.get(key)
- if task_id is not None:
- self.progress.advance(task_id)
- else:
- self.failure_types[key] = self.progress.add_task(key, total=self.failed_files, completed=1)
- self.update_total(self.failed_files)
-
- def return_totals(self) -> list[UiFailureTotal]:
- """Returns the total number of failed sites and reasons."""
- failures = {}
- for key, task_id in self.failure_types.items():
- task = next(task for task in self.progress.tasks if task.id == task_id)
- failures[key] = task.completed
- return sorted(UiFailureTotal.from_pair(*f) for f in failures.items())
-
-
-class DownloadStatsProgress(StatsProgress):
- """Class that keeps track of download failures and reasons."""
-
-
-class ScrapeStatsProgress(StatsProgress):
- """Class that keeps track of scraping failures and reasons."""
-
- title = "Scrape Failures"
-
- def __init__(self) -> None:
- super().__init__()
- self.unsupported_urls = 0
- self.sent_to_jdownloader = 0
- self.unsupported_urls_skipped = 0
-
- def add_unsupported(self, sent_to_jdownloader: bool = False) -> None:
- """Adds an unsupported url to the progress bar."""
- self.unsupported_urls += 1
- if sent_to_jdownloader:
- self.sent_to_jdownloader += 1
- else:
- self.unsupported_urls_skipped += 1
-
-
-def get_pretty_failure(failure: str) -> str:
- with contextlib.suppress(KeyError):
- return FAILURE_OVERRIDES[failure]
- return capitalize_words(failure)
-
-
-def capitalize_words(text: str) -> str:
- """Capitalize first letter of each word
-
- Unlike `str.capwords()`, this only caps the first letter of each word without modifying the rest of the word"""
- return " ".join([capitalize_first_letter(word) for word in text.split()])
-
-
-def capitalize_first_letter(word: str) -> str:
- return word[0].capitalize() + word[1:]
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index f7e9b3c55..100670151 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -145,7 +145,7 @@ def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yar
logger.error(f"Scrape Failed: {link_to_show} ({error_log_msg.main_log_msg})", exc_info=exc_info)
self.manager.logs.write_scrape_error(link_to_show, error_log_msg.csv_log_msg, origin)
- self.manager.progress_manager.scrape_stats_progress.add_failure(error_log_msg.ui_failure)
+ self.manager.scrape_mapper.tui.scrape_errors.add(error_log_msg.ui_failure)
@overload
diff --git a/tests/crawlers/test_xenforo.py b/tests/crawlers/test_xenforo.py
index 03930229b..10736db48 100644
--- a/tests/crawlers/test_xenforo.py
+++ b/tests/crawlers/test_xenforo.py
@@ -18,13 +18,8 @@ def _item(url: str) -> ScrapeItem:
return ScrapeItem(url=AbsoluteHttpURL(url))
-class MockProgress:
- scraping_progress = None
-
-
manager = Manager()
-scrape_item = _item("https://xenforo.com/community")
-manager.progress_manager = MockProgress() # type: ignore
+scrape_item = _item("https://xenforo.com/community") # type: ignore
crawler_instances = {crawler: crawler(manager) for crawler in crawlers.XF_CRAWLERS}
TEST_CRAWLER = crawler_instances[crawlers.CelebForumCrawler]
diff --git a/tests/test_startup.py b/tests/test_startup.py
index dcf4c370b..27a270e37 100644
--- a/tests/test_startup.py
+++ b/tests/test_startup.py
@@ -1,7 +1,7 @@
import pytest
+from cyberdrop_dl import program_ui
from cyberdrop_dl.__main__ import main
-from cyberdrop_dl.ui import program_ui
def test_startup(capsys: pytest.CaptureFixture[str]) -> None: