Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: some type errors (#1199)
NTFSvolume4f2e80ab5242
diff --git a/cyberdrop_dl/constants.py b/cyberdrop_dl/constants.pyindex 8330223c2..fe8acd663 100644--- a/cyberdrop_dl/constants.py+++ b/cyberdrop_dl/constants.py@@ -91,9 +91,9 @@ class Hashing(StrEnum):POST_DOWNLOAD = auto()@classmethod- def _missing_(cls, value: str) -> "Hashing":+ def _missing_(cls, value: object) -> "Hashing":try:- return cls[str(value.upper())]+ return cls[str(value).upper()]except KeyError as e:raise ediff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.pyindex f6a5ac671..666786184 100644--- a/cyberdrop_dl/crawlers/_yetishare.py+++ b/cyberdrop_dl/crawlers/_yetishare.py@@ -77,6 +77,7 @@ class YetiShareCrawler(Crawler, is_abc=True):album_id = scrape_item.url.parts[2]title: str = ""default_data = {"pageType": "folder", "perPage": 0, "filterOrderBy": ""}+ n_pages = 1for page in itertools.count(1):data = default_data | {"nodeId": node_id, "pageStart": page}ajax_soup, ajax_title = await self.get_soup_from_ajax(data, scrape_item)@@ -101,6 +102,7 @@ class YetiShareCrawler(Crawler, is_abc=True):album_id = scrape_item.url.parts[2]page = 1default_data = {"pageType": "nonaccountshared", "perPage": 0, "filterOrderBy": ""}+ n_pages = 1while True:data = default_data | {"nodeId": node_id, "pageStart": page}ajax_soup, ajax_title = await self.get_soup_from_ajax(data, scrape_item)diff --git a/cyberdrop_dl/crawlers/camwhores_dot_tv.py b/cyberdrop_dl/crawlers/camwhores_dot_tv.pyindex d98f93caa..a559fae16 100644--- a/cyberdrop_dl/crawlers/camwhores_dot_tv.py+++ b/cyberdrop_dl/crawlers/camwhores_dot_tv.py@@ -39,7 +39,9 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler):video = super().get_video_info(soup)return video._replace(title=open_graph.title(soup))- def parse_url(self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *_) -> AbsoluteHttpURL:+ def parse_url(+ self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = False+ ) -> AbsoluteHttpURL:return super().parse_url(link_str, relative_to, trim=False)async def picture(self, scrape_item: ScrapeItem) -> None:diff --git a/cyberdrop_dl/crawlers/coomer.py b/cyberdrop_dl/crawlers/coomer.pyindex 39eb29fce..e16e8f3f3 100644--- a/cyberdrop_dl/crawlers/coomer.py+++ b/cyberdrop_dl/crawlers/coomer.py@@ -6,7 +6,7 @@ from aiolimiter import AsyncLimiterfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from ._kemono_base import KemonoBaseCrawler, UserPost+from ._kemono_base import KemonoBaseCrawler, Postif TYPE_CHECKING:from aiohttp_client_cache.response import AnyResponse@@ -40,7 +40,7 @@ class CoomerCrawler(KemonoBaseCrawler):async def fetch(self, scrape_item: ScrapeItem) -> None:return await self._fetch_kemono_defaults(scrape_item)- def _handle_post_content(self, scrape_item: ScrapeItem, post: UserPost) -> None:+ def _handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:"""Handles the content of a post."""if "#ad" in post.content and self.manager.config_manager.settings_data.ignore_options.ignore_coomer_ads:returndiff --git a/cyberdrop_dl/crawlers/discourse/_discourse.py b/cyberdrop_dl/crawlers/discourse/_discourse.pyindex 7a0681abc..7493038e4 100644--- a/cyberdrop_dl/crawlers/discourse/_discourse.py+++ b/cyberdrop_dl/crawlers/discourse/_discourse.py@@ -47,10 +47,11 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):def is_attachment(link: AbsoluteHttpURL) -> bool:return "uploads" in link.parts- async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:- if len(scrape_item.url.parts) < 5 and not scrape_item.url.suffix:+ async def handle_internal_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:+ link = link or scrape_item.url+ if len(link.parts) < 5 and not link.suffix:return- await super().handle_internal_link(scrape_item)+ await super().handle_internal_link(scrape_item, link)async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:@@ -89,11 +90,11 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):async def forum(self, scrape_item: ScrapeItem) -> None:raise NotImplementedError- def parse_thread(self, *args) -> None:+ def parse_thread(self, *args, **kwargs) -> None:raise NotImplementedError- async def thread(self, scrape_item: ScrapeItem, topic: Topic) -> None:- await self.topic(scrape_item, topic)+ async def thread(self, scrape_item: ScrapeItem, thread: Topic) -> None:+ await self.topic(scrape_item, thread)@error_handling_wrapperasync def process_posts(self, scrape_item: ScrapeItem, topic: Topic) -> None:@@ -147,8 +148,10 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):return unique(iter_links())- def parse_url(self, link: str) -> AbsoluteHttpURL:- return _clean_url(super().parse_url(link))+ def parse_url(+ self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True+ ) -> AbsoluteHttpURL:+ return _clean_url(super().parse_url(link_str, relative_to, trim=trim))def _clean_url(url: AbsoluteHttpURL) -> AbsoluteHttpURL:diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.pyindex f17b1e9ec..d99b72ed5 100644--- a/cyberdrop_dl/crawlers/google_drive.py+++ b/cyberdrop_dl/crawlers/google_drive.py@@ -146,7 +146,7 @@ class GoogleDriveCrawler(Crawler):await self.handle_file(canonical_url, scrape_item, filename, ext, debrid_link=link)async def get_file_url_and_name(self, url: AbsoluteHttpURL, file_id: str) -> tuple[AbsoluteHttpURL, str | None]:- soup = last_error = None+ soup = last_error = response = Nonecurrent_url: AbsoluteHttpURL | None = urltry_file_open_url = True@@ -173,7 +173,7 @@ class GoogleDriveCrawler(Crawler):current_url = docs_urlcontinue- if response.content_disposition and response.content_disposition.filename:+ if response and response.content_disposition and response.content_disposition.filename:return current_url, response.content_disposition.filenamecurrent_url = self.get_url_from_download_button(soup)diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.pyindex 3c9aa686c..47cf146e8 100644--- a/cyberdrop_dl/crawlers/imgbb.py+++ b/cyberdrop_dl/crawlers/imgbb.py@@ -46,6 +46,7 @@ class ImgBBCrawler(Crawler):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:title: str = ""+ first_page = Noneasync for soup in self.web_pager(scrape_item.url / "sub"):if not title:@@ -59,6 +60,7 @@ class ImgBBCrawler(Crawler):for _, sub_album in self.iter_children(scrape_item, soup, ALBUM_PAGE_SELECTOR):self.manager.task_group.create_task(self.run(sub_album))+ assert first_pageasync for soup in self.web_pager(first_page):for _, image in self.iter_children(scrape_item, soup, IMAGE_PAGE_SELECTOR):self.manager.task_group.create_task(self.run(image))diff --git a/cyberdrop_dl/crawlers/nekohouse.py b/cyberdrop_dl/crawlers/nekohouse.pyindex 37e9d2dd6..758300a09 100644--- a/cyberdrop_dl/crawlers/nekohouse.py+++ b/cyberdrop_dl/crawlers/nekohouse.py@@ -7,12 +7,11 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom ._kemono_base import KemonoBaseCrawlerif TYPE_CHECKING:- from cyberdrop_dl.crawlers.crawler import SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import ScrapeItemclass NekohouseCrawler(KemonoBaseCrawler):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ SUPPORTED_PATHS: ClassVar[dict[str, str]] = {"Model": "/<service>/user/<user>","Individual Post": "/<service>/<user>/post/...","Direct links": "",diff --git a/cyberdrop_dl/crawlers/reddit.py b/cyberdrop_dl/crawlers/reddit.pyindex e32c25f8f..7187106cb 100644--- a/cyberdrop_dl/crawlers/reddit.py+++ b/cyberdrop_dl/crawlers/reddit.py@@ -24,7 +24,6 @@ if TYPE_CHECKING:class MediaFile(TypedDict):y: int # Heightx: int # Width- u: str # URLclass MediaSource(MediaFile):diff --git a/cyberdrop_dl/crawlers/sex_dot_com.py b/cyberdrop_dl/crawlers/sex_dot_com.pyindex 2d4fe4ef0..27fe6e556 100644--- a/cyberdrop_dl/crawlers/sex_dot_com.py+++ b/cyberdrop_dl/crawlers/sex_dot_com.py@@ -76,6 +76,8 @@ class SexDotComCrawler(Crawler):elif fileType.startswith("video"):media_url = AbsoluteHttpURL(real_item["sources"][0]["fullPath"])filename, ext = self.get_filename_and_ext(media_url.name)+ else:+ returnscrape_item.possible_datetime = self.parse_date(real_item["createdAt"])scrape_item.url = canonical_urldiff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.pyindex f22bcfc5b..bfaae8e33 100644--- a/cyberdrop_dl/crawlers/spankbang.py+++ b/cyberdrop_dl/crawlers/spankbang.py@@ -134,7 +134,7 @@ class SpankBangCrawler(Crawler):scrape_item.possible_datetime = self.parse_iso_date(video.date)link = self.parse_url(link_str)filename, ext = self.get_filename_and_ext(link.name)- custom_filename = self.create_custom_filename(video.title, ext, file_id=video_id, resolution=resolution)+ custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=resolution)await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)diff --git a/cyberdrop_dl/crawlers/xxxbunker.py b/cyberdrop_dl/crawlers/xxxbunker.pyindex d3af6a61a..64e5178f9 100644--- a/cyberdrop_dl/crawlers/xxxbunker.py+++ b/cyberdrop_dl/crawlers/xxxbunker.py@@ -137,9 +137,10 @@ class XXXBunkerCrawler(Crawler):while True:attempt = 1rate_limited = True+ soup = Nonewhile rate_limited and attempt <= MAX_RETRIES:async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, page_url)+ soup = await self.client.get_soup(self.DOMAIN, page_url)await asyncio.sleep(self.wait_time)if "TRAFFIC VERIFICATION" not in soup.text:@@ -153,7 +154,7 @@ class XXXBunkerCrawler(Crawler):if rate_limited:raise ScrapeError(429)-+ assert soupyield souppage_url_str = css.select_one_get_attr_or_none(soup, NEXT_PAGE_SELECTOR, "href")if not page_url_str:diff --git a/cyberdrop_dl/data_structures/hash.py b/cyberdrop_dl/data_structures/hash.pyindex 47cf42aed..2f5379ee7 100644--- a/cyberdrop_dl/data_structures/hash.py+++ b/cyberdrop_dl/data_structures/hash.py@@ -7,8 +7,8 @@ class Hashing(StrEnum):POST_DOWNLOAD = auto()@classmethod- def _missing_(cls, value: str):+ def _missing_(cls, value: object):try:- return cls[str(value.upper())]+ return cls[str(value).upper()]except KeyError as e:raise ediff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex 8773eb046..3d0773bc3 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -1,3 +1,4 @@+# type: ignore[reportIncompatibleVariableOverride]from __future__ import annotationsimport copydiff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.pyindex 7a9ad2977..5c1c2ceb0 100644--- a/cyberdrop_dl/director.py+++ b/cyberdrop_dl/director.py@@ -314,8 +314,8 @@ def _setup_manager(args: tuple[str, ...] | None = None) -> Manager:After this function returns, the manager will be ready to use and scraping / downloading can begin."""with _startup_logging(first_time_setup=True):+ manager = Manager(args)try:- manager = Manager(args)manager.startup()if manager.parsed_args.cli_only_args.multiconfig:startup_logger.info("validating all configs, please wait...")diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex 512410051..56331801e 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -183,7 +183,6 @@ class Node(TypedDict):a: str # attributes (within this: 'n' Name)k: str # keyu: str # user id- s: int # sizets: int # creation date (timestamp)g: NotRequired[str] # Access URL@@ -446,7 +445,7 @@ async def generate_hashcash_token(challenge: str) -> str:def get_decrypt_data(node_type: NodeType, full_key: U32IntTupleArray) -> DecryptData:if node_type == NodeType.FILE:k = (full_key[0] ^ full_key[4], full_key[1] ^ full_key[5], full_key[2] ^ full_key[6], full_key[3] ^ full_key[7])- elif node_type == NodeType.FOLDER:+ else:k = full_keyiv: U32IntTupleArray = (*full_key[4:6], 0, 0)@@ -844,14 +843,15 @@ class MegaDownloadClient(DownloadClient):class MegaDownloader(Downloader):+ client: MegaDownloadClient+def __init__(self, manager: Manager, domain: str) -> None:super().__init__(manager, domain)self.api = MegaApi(manager)- self.client: MegaDownloadClientdef startup(self) -> None:"""Starts the downloader."""- self.client = MegaDownloadClient(self.manager)+ self.client = MegaDownloadClient(self.manager) # type: ignore[reportIncompatibleVariableOverride]self._semaphore = asyncio.Semaphore(self.manager.download_manager.get_download_limit(self.domain))def register(self, url: URL, crypto: DecryptData) -> None:diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.pyindex e23765634..fd76e5e82 100644--- a/cyberdrop_dl/managers/progress_manager.py+++ b/cyberdrop_dl/managers/progress_manager.py@@ -202,7 +202,7 @@ def log_failures(failures: list[UiFailureTotal], title: str = "Failures:", last_error_padding = max(len(str(max(error_codes))), error_padding)for f in failures:error = f.error_code if f.error_code is not None else ""- log_red(f" {error:>{error_padding}}{' ' if error_padding else ''}{f.msg}: {f.count:,}")+ log_red(f" {error:>{error_padding}}{' ' if error_padding else ''}{f.msg}: {f.total:,}")return error_paddingdiff --git a/cyberdrop_dl/scraper/jdownloader.py b/cyberdrop_dl/scraper/jdownloader.pyindex bb31e2d10..d3c9fd94d 100644--- a/cyberdrop_dl/scraper/jdownloader.py+++ b/cyberdrop_dl/scraper/jdownloader.py@@ -60,7 +60,7 @@ class JDownloader:if not self.enabled or self._agent is not None:returntry:- self._connect()+ return self._connect()except JDownloaderError as e:msg = e.messageexcept myjdapi.MYJDDeviceNotFoundException:diff --git a/cyberdrop_dl/ui/program_ui.py b/cyberdrop_dl/ui/program_ui.pyindex c02e7a043..2813c997f 100644--- a/cyberdrop_dl/ui/program_ui.py+++ b/cyberdrop_dl/ui/program_ui.py@@ -301,6 +301,7 @@ class ProgramUI:def vacuum_database(db_path: Path) -> None:if not db_path.is_file():return+ conn = Nonetry:conn = sqlite3.connect(db_path)cursor = conn.cursor()diff --git a/cyberdrop_dl/ui/progress/file_progress.py b/cyberdrop_dl/ui/progress/file_progress.pyindex ad155931f..8bca91d39 100644--- a/cyberdrop_dl/ui/progress/file_progress.py+++ b/cyberdrop_dl/ui/progress/file_progress.py@@ -56,7 +56,7 @@ class FileProgress(DequeProgress):return total- def add_task(self, *, domain: str, filename: str, expected_size: int | None = None) -> TaskID:+ def add_task(self, *, domain: str, filename: str, expected_size: int | None = None) -> TaskID: # type: ignore[reportIncompatibleMethodOverride]"""Adds a new task to the progress bar."""filename = filename.split("/")[-1].encode("ascii", "ignore").decode().strip()description = escape(adjust_title(filename, length=40))diff --git a/cyberdrop_dl/ui/progress/scraping_progress.py b/cyberdrop_dl/ui/progress/scraping_progress.pyindex 0034bd190..3b8037dee 100644--- a/cyberdrop_dl/ui/progress/scraping_progress.py+++ b/cyberdrop_dl/ui/progress/scraping_progress.py@@ -41,6 +41,6 @@ class ScrapingProgress(DequeProgress):if not passed:self.manager.progress_manager.file_progress.redraw()- def add_task(self, url: URL) -> TaskID:+ def add_task(self, url: URL) -> TaskID: # type: ignore[reportIncompatibleMethodOverride]"""Adds a new task to the progress bar."""return super().add_task(str(url))diff --git a/cyberdrop_dl/ui/progress/statistic_progress.py b/cyberdrop_dl/ui/progress/statistic_progress.pyindex ff1537947..9459a68dd 100644--- a/cyberdrop_dl/ui/progress/statistic_progress.py+++ b/cyberdrop_dl/ui/progress/statistic_progress.py@@ -41,17 +41,17 @@ class TaskInfo(NamedTuple):class UiFailureTotal(NamedTuple):full_msg: str- count: int+ total: interror_code: int | Nonemsg: str@classmethod- def from_pair(cls, full_msg: str, count: int) -> UiFailureTotal:+ def from_pair(cls, full_msg: str, total: int) -> UiFailureTotal:parts = full_msg.split(" ", 1)if len(parts) > 1 and parts[0].isdigit():error_code, msg = parts- return cls(full_msg, count, int(error_code), msg)- return cls(full_msg, count, None, full_msg)+ return cls(full_msg, total, int(error_code), msg)+ return cls(full_msg, total, None, full_msg)def get_tasks_info_sorted(progress: Progress) -> tuple[list[TaskInfo], bool]:diff --git a/cyberdrop_dl/utils/ffmpeg.py b/cyberdrop_dl/utils/ffmpeg.pyindex e137abba9..52ecb8dca 100644--- a/cyberdrop_dl/utils/ffmpeg.py+++ b/cyberdrop_dl/utils/ffmpeg.py@@ -242,7 +242,7 @@ class Stream:return self.codec_name@classmethod- def validate(cls, stream_info: Mapping[str, Any]) -> dict[str, Any]:+ def validate(cls, stream_info: StreamDict) -> dict[str, Any]:info = get_valid_dict(cls, stream_info)tags = Tags(CIMultiDict(stream_info.get("tags", {})))duration: float | str | None = stream_info.get("duration") or tags.get("duration")@@ -251,7 +251,7 @@ class Stream:return info | {"tags": tags, "duration": duration, "bitrate": bitrate}@classmethod- def from_dict(cls, stream_info: Mapping[str, Any]) -> Self:+ def from_dict(cls, stream_info: StreamDict) -> Self:return cls(**cls.validate(stream_info))def as_dict(self) -> dict[str, Any]:diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.pyindex 4d61792d4..5b607f2f2 100644--- a/cyberdrop_dl/utils/logger.py+++ b/cyberdrop_dl/utils/logger.py@@ -108,7 +108,7 @@ class QueuedLogger:class NoPaddingLogRender(LogRender):cdl_padding = 0- def __call__(+ def __call__( # type: ignore[reportIncompatibleMethodOverride]self,console: Console,renderables: Iterable[ConsoleRenderable],