Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: some type errors (#1199)

NTFSvolume4f2e80ab5242
diff --git a/cyberdrop_dl/constants.py b/cyberdrop_dl/constants.py
index 8330223c2..fe8acd663 100644
--- a/cyberdrop_dl/constants.py
+++ b/cyberdrop_dl/constants.py
@@ -91,9 +91,9 @@ class Hashing(StrEnum):
POST_DOWNLOAD = auto()
@classmethod
- def _missing_(cls, value: str) -> "Hashing":
+ def _missing_(cls, value: object) -> "Hashing":
try:
- return cls[str(value.upper())]
+ return cls[str(value).upper()]
except KeyError as e:
raise e
diff --git a/cyberdrop_dl/crawlers/_yetishare.py b/cyberdrop_dl/crawlers/_yetishare.py
index f6a5ac671..666786184 100644
--- a/cyberdrop_dl/crawlers/_yetishare.py
+++ b/cyberdrop_dl/crawlers/_yetishare.py
@@ -77,6 +77,7 @@ class YetiShareCrawler(Crawler, is_abc=True):
album_id = scrape_item.url.parts[2]
title: str = ""
default_data = {"pageType": "folder", "perPage": 0, "filterOrderBy": ""}
+ n_pages = 1
for page in itertools.count(1):
data = default_data | {"nodeId": node_id, "pageStart": page}
ajax_soup, ajax_title = await self.get_soup_from_ajax(data, scrape_item)
@@ -101,6 +102,7 @@ class YetiShareCrawler(Crawler, is_abc=True):
album_id = scrape_item.url.parts[2]
page = 1
default_data = {"pageType": "nonaccountshared", "perPage": 0, "filterOrderBy": ""}
+ n_pages = 1
while True:
data = default_data | {"nodeId": node_id, "pageStart": page}
ajax_soup, ajax_title = await self.get_soup_from_ajax(data, scrape_item)
diff --git a/cyberdrop_dl/crawlers/camwhores_dot_tv.py b/cyberdrop_dl/crawlers/camwhores_dot_tv.py
index d98f93caa..a559fae16 100644
--- a/cyberdrop_dl/crawlers/camwhores_dot_tv.py
+++ b/cyberdrop_dl/crawlers/camwhores_dot_tv.py
@@ -39,7 +39,9 @@ class CamwhoresTVCrawler(KernelVideoSharingCrawler):
video = super().get_video_info(soup)
return video._replace(title=open_graph.title(soup))
- def parse_url(self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *_) -> AbsoluteHttpURL:
+ def parse_url(
+ self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = False
+ ) -> AbsoluteHttpURL:
return super().parse_url(link_str, relative_to, trim=False)
async def picture(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/crawlers/coomer.py b/cyberdrop_dl/crawlers/coomer.py
index 39eb29fce..e16e8f3f3 100644
--- a/cyberdrop_dl/crawlers/coomer.py
+++ b/cyberdrop_dl/crawlers/coomer.py
@@ -6,7 +6,7 @@ from aiolimiter import AsyncLimiter
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from ._kemono_base import KemonoBaseCrawler, UserPost
+from ._kemono_base import KemonoBaseCrawler, Post
if TYPE_CHECKING:
from aiohttp_client_cache.response import AnyResponse
@@ -40,7 +40,7 @@ class CoomerCrawler(KemonoBaseCrawler):
async def fetch(self, scrape_item: ScrapeItem) -> None:
return await self._fetch_kemono_defaults(scrape_item)
- def _handle_post_content(self, scrape_item: ScrapeItem, post: UserPost) -> None:
+ def _handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:
"""Handles the content of a post."""
if "#ad" in post.content and self.manager.config_manager.settings_data.ignore_options.ignore_coomer_ads:
return
diff --git a/cyberdrop_dl/crawlers/discourse/_discourse.py b/cyberdrop_dl/crawlers/discourse/_discourse.py
index 7a0681abc..7493038e4 100644
--- a/cyberdrop_dl/crawlers/discourse/_discourse.py
+++ b/cyberdrop_dl/crawlers/discourse/_discourse.py
@@ -47,10 +47,11 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
def is_attachment(link: AbsoluteHttpURL) -> bool:
return "uploads" in link.parts
- async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:
- if len(scrape_item.url.parts) < 5 and not scrape_item.url.suffix:
+ async def handle_internal_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:
+ link = link or scrape_item.url
+ if len(link.parts) < 5 and not link.suffix:
return
- await super().handle_internal_link(scrape_item)
+ await super().handle_internal_link(scrape_item, link)
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
@@ -89,11 +90,11 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
async def forum(self, scrape_item: ScrapeItem) -> None:
raise NotImplementedError
- def parse_thread(self, *args) -> None:
+ def parse_thread(self, *args, **kwargs) -> None:
raise NotImplementedError
- async def thread(self, scrape_item: ScrapeItem, topic: Topic) -> None:
- await self.topic(scrape_item, topic)
+ async def thread(self, scrape_item: ScrapeItem, thread: Topic) -> None:
+ await self.topic(scrape_item, thread)
@error_handling_wrapper
async def process_posts(self, scrape_item: ScrapeItem, topic: Topic) -> None:
@@ -147,8 +148,10 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
return unique(iter_links())
- def parse_url(self, link: str) -> AbsoluteHttpURL:
- return _clean_url(super().parse_url(link))
+ def parse_url(
+ self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True
+ ) -> AbsoluteHttpURL:
+ return _clean_url(super().parse_url(link_str, relative_to, trim=trim))
def _clean_url(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.py
index f17b1e9ec..d99b72ed5 100644
--- a/cyberdrop_dl/crawlers/google_drive.py
+++ b/cyberdrop_dl/crawlers/google_drive.py
@@ -146,7 +146,7 @@ class GoogleDriveCrawler(Crawler):
await self.handle_file(canonical_url, scrape_item, filename, ext, debrid_link=link)
async def get_file_url_and_name(self, url: AbsoluteHttpURL, file_id: str) -> tuple[AbsoluteHttpURL, str | None]:
- soup = last_error = None
+ soup = last_error = response = None
current_url: AbsoluteHttpURL | None = url
try_file_open_url = True
@@ -173,7 +173,7 @@ class GoogleDriveCrawler(Crawler):
current_url = docs_url
continue
- if response.content_disposition and response.content_disposition.filename:
+ if response and response.content_disposition and response.content_disposition.filename:
return current_url, response.content_disposition.filename
current_url = self.get_url_from_download_button(soup)
diff --git a/cyberdrop_dl/crawlers/imgbb.py b/cyberdrop_dl/crawlers/imgbb.py
index 3c9aa686c..47cf146e8 100644
--- a/cyberdrop_dl/crawlers/imgbb.py
+++ b/cyberdrop_dl/crawlers/imgbb.py
@@ -46,6 +46,7 @@ class ImgBBCrawler(Crawler):
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
title: str = ""
+ first_page = None
async for soup in self.web_pager(scrape_item.url / "sub"):
if not title:
@@ -59,6 +60,7 @@ class ImgBBCrawler(Crawler):
for _, sub_album in self.iter_children(scrape_item, soup, ALBUM_PAGE_SELECTOR):
self.manager.task_group.create_task(self.run(sub_album))
+ assert first_page
async for soup in self.web_pager(first_page):
for _, image in self.iter_children(scrape_item, soup, IMAGE_PAGE_SELECTOR):
self.manager.task_group.create_task(self.run(image))
diff --git a/cyberdrop_dl/crawlers/nekohouse.py b/cyberdrop_dl/crawlers/nekohouse.py
index 37e9d2dd6..758300a09 100644
--- a/cyberdrop_dl/crawlers/nekohouse.py
+++ b/cyberdrop_dl/crawlers/nekohouse.py
@@ -7,12 +7,11 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from ._kemono_base import KemonoBaseCrawler
if TYPE_CHECKING:
- from cyberdrop_dl.crawlers.crawler import SupportedPaths
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
class NekohouseCrawler(KemonoBaseCrawler):
- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ SUPPORTED_PATHS: ClassVar[dict[str, str]] = {
"Model": "/<service>/user/<user>",
"Individual Post": "/<service>/<user>/post/...",
"Direct links": "",
diff --git a/cyberdrop_dl/crawlers/reddit.py b/cyberdrop_dl/crawlers/reddit.py
index e32c25f8f..7187106cb 100644
--- a/cyberdrop_dl/crawlers/reddit.py
+++ b/cyberdrop_dl/crawlers/reddit.py
@@ -24,7 +24,6 @@ if TYPE_CHECKING:
class MediaFile(TypedDict):
y: int # Height
x: int # Width
- u: str # URL
class MediaSource(MediaFile):
diff --git a/cyberdrop_dl/crawlers/sex_dot_com.py b/cyberdrop_dl/crawlers/sex_dot_com.py
index 2d4fe4ef0..27fe6e556 100644
--- a/cyberdrop_dl/crawlers/sex_dot_com.py
+++ b/cyberdrop_dl/crawlers/sex_dot_com.py
@@ -76,6 +76,8 @@ class SexDotComCrawler(Crawler):
elif fileType.startswith("video"):
media_url = AbsoluteHttpURL(real_item["sources"][0]["fullPath"])
filename, ext = self.get_filename_and_ext(media_url.name)
+ else:
+ return
scrape_item.possible_datetime = self.parse_date(real_item["createdAt"])
scrape_item.url = canonical_url
diff --git a/cyberdrop_dl/crawlers/spankbang.py b/cyberdrop_dl/crawlers/spankbang.py
index f22bcfc5b..bfaae8e33 100644
--- a/cyberdrop_dl/crawlers/spankbang.py
+++ b/cyberdrop_dl/crawlers/spankbang.py
@@ -134,7 +134,7 @@ class SpankBangCrawler(Crawler):
scrape_item.possible_datetime = self.parse_iso_date(video.date)
link = self.parse_url(link_str)
filename, ext = self.get_filename_and_ext(link.name)
- custom_filename = self.create_custom_filename(video.title, ext, file_id=video_id, resolution=resolution)
+ custom_filename = self.create_custom_filename(video.title, ext, file_id=video.id, resolution=resolution)
await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
diff --git a/cyberdrop_dl/crawlers/xxxbunker.py b/cyberdrop_dl/crawlers/xxxbunker.py
index d3af6a61a..64e5178f9 100644
--- a/cyberdrop_dl/crawlers/xxxbunker.py
+++ b/cyberdrop_dl/crawlers/xxxbunker.py
@@ -137,9 +137,10 @@ class XXXBunkerCrawler(Crawler):
while True:
attempt = 1
rate_limited = True
+ soup = None
while rate_limited and attempt <= MAX_RETRIES:
async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, page_url)
+ soup = await self.client.get_soup(self.DOMAIN, page_url)
await asyncio.sleep(self.wait_time)
if "TRAFFIC VERIFICATION" not in soup.text:
@@ -153,7 +154,7 @@ class XXXBunkerCrawler(Crawler):
if rate_limited:
raise ScrapeError(429)
-
+ assert soup
yield soup
page_url_str = css.select_one_get_attr_or_none(soup, NEXT_PAGE_SELECTOR, "href")
if not page_url_str:
diff --git a/cyberdrop_dl/data_structures/hash.py b/cyberdrop_dl/data_structures/hash.py
index 47cf42aed..2f5379ee7 100644
--- a/cyberdrop_dl/data_structures/hash.py
+++ b/cyberdrop_dl/data_structures/hash.py
@@ -7,8 +7,8 @@ class Hashing(StrEnum):
POST_DOWNLOAD = auto()
@classmethod
- def _missing_(cls, value: str):
+ def _missing_(cls, value: object):
try:
- return cls[str(value.upper())]
+ return cls[str(value).upper()]
except KeyError as e:
raise e
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index 8773eb046..3d0773bc3 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -1,3 +1,4 @@
+# type: ignore[reportIncompatibleVariableOverride]
from __future__ import annotations
import copy
diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.py
index 7a9ad2977..5c1c2ceb0 100644
--- a/cyberdrop_dl/director.py
+++ b/cyberdrop_dl/director.py
@@ -314,8 +314,8 @@ def _setup_manager(args: tuple[str, ...] | None = None) -> Manager:
After this function returns, the manager will be ready to use and scraping / downloading can begin.
"""
with _startup_logging(first_time_setup=True):
+ manager = Manager(args)
try:
- manager = Manager(args)
manager.startup()
if manager.parsed_args.cli_only_args.multiconfig:
startup_logger.info("validating all configs, please wait...")
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index 512410051..56331801e 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -183,7 +183,6 @@ class Node(TypedDict):
a: str # attributes (within this: 'n' Name)
k: str # key
u: str # user id
- s: int # size
ts: int # creation date (timestamp)
g: NotRequired[str] # Access URL
@@ -446,7 +445,7 @@ async def generate_hashcash_token(challenge: str) -> str:
def get_decrypt_data(node_type: NodeType, full_key: U32IntTupleArray) -> DecryptData:
if node_type == NodeType.FILE:
k = (full_key[0] ^ full_key[4], full_key[1] ^ full_key[5], full_key[2] ^ full_key[6], full_key[3] ^ full_key[7])
- elif node_type == NodeType.FOLDER:
+ else:
k = full_key
iv: U32IntTupleArray = (*full_key[4:6], 0, 0)
@@ -844,14 +843,15 @@ class MegaDownloadClient(DownloadClient):
class MegaDownloader(Downloader):
+ client: MegaDownloadClient
+
def __init__(self, manager: Manager, domain: str) -> None:
super().__init__(manager, domain)
self.api = MegaApi(manager)
- self.client: MegaDownloadClient
def startup(self) -> None:
"""Starts the downloader."""
- self.client = MegaDownloadClient(self.manager)
+ self.client = MegaDownloadClient(self.manager) # type: ignore[reportIncompatibleVariableOverride]
self._semaphore = asyncio.Semaphore(self.manager.download_manager.get_download_limit(self.domain))
def register(self, url: URL, crypto: DecryptData) -> None:
diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.py
index e23765634..fd76e5e82 100644
--- a/cyberdrop_dl/managers/progress_manager.py
+++ b/cyberdrop_dl/managers/progress_manager.py
@@ -202,7 +202,7 @@ def log_failures(failures: list[UiFailureTotal], title: str = "Failures:", last_
error_padding = max(len(str(max(error_codes))), error_padding)
for f in failures:
error = f.error_code if f.error_code is not None else ""
- log_red(f" {error:>{error_padding}}{' ' if error_padding else ''}{f.msg}: {f.count:,}")
+ log_red(f" {error:>{error_padding}}{' ' if error_padding else ''}{f.msg}: {f.total:,}")
return error_padding
diff --git a/cyberdrop_dl/scraper/jdownloader.py b/cyberdrop_dl/scraper/jdownloader.py
index bb31e2d10..d3c9fd94d 100644
--- a/cyberdrop_dl/scraper/jdownloader.py
+++ b/cyberdrop_dl/scraper/jdownloader.py
@@ -60,7 +60,7 @@ class JDownloader:
if not self.enabled or self._agent is not None:
return
try:
- self._connect()
+ return self._connect()
except JDownloaderError as e:
msg = e.message
except myjdapi.MYJDDeviceNotFoundException:
diff --git a/cyberdrop_dl/ui/program_ui.py b/cyberdrop_dl/ui/program_ui.py
index c02e7a043..2813c997f 100644
--- a/cyberdrop_dl/ui/program_ui.py
+++ b/cyberdrop_dl/ui/program_ui.py
@@ -301,6 +301,7 @@ class ProgramUI:
def vacuum_database(db_path: Path) -> None:
if not db_path.is_file():
return
+ conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
diff --git a/cyberdrop_dl/ui/progress/file_progress.py b/cyberdrop_dl/ui/progress/file_progress.py
index ad155931f..8bca91d39 100644
--- a/cyberdrop_dl/ui/progress/file_progress.py
+++ b/cyberdrop_dl/ui/progress/file_progress.py
@@ -56,7 +56,7 @@ class FileProgress(DequeProgress):
return total
- def add_task(self, *, domain: str, filename: str, expected_size: int | None = None) -> TaskID:
+ def add_task(self, *, domain: str, filename: str, expected_size: int | None = None) -> TaskID: # type: ignore[reportIncompatibleMethodOverride]
"""Adds a new task to the progress bar."""
filename = filename.split("/")[-1].encode("ascii", "ignore").decode().strip()
description = escape(adjust_title(filename, length=40))
diff --git a/cyberdrop_dl/ui/progress/scraping_progress.py b/cyberdrop_dl/ui/progress/scraping_progress.py
index 0034bd190..3b8037dee 100644
--- a/cyberdrop_dl/ui/progress/scraping_progress.py
+++ b/cyberdrop_dl/ui/progress/scraping_progress.py
@@ -41,6 +41,6 @@ class ScrapingProgress(DequeProgress):
if not passed:
self.manager.progress_manager.file_progress.redraw()
- def add_task(self, url: URL) -> TaskID:
+ def add_task(self, url: URL) -> TaskID: # type: ignore[reportIncompatibleMethodOverride]
"""Adds a new task to the progress bar."""
return super().add_task(str(url))
diff --git a/cyberdrop_dl/ui/progress/statistic_progress.py b/cyberdrop_dl/ui/progress/statistic_progress.py
index ff1537947..9459a68dd 100644
--- a/cyberdrop_dl/ui/progress/statistic_progress.py
+++ b/cyberdrop_dl/ui/progress/statistic_progress.py
@@ -41,17 +41,17 @@ class TaskInfo(NamedTuple):
class UiFailureTotal(NamedTuple):
full_msg: str
- count: int
+ total: int
error_code: int | None
msg: str
@classmethod
- def from_pair(cls, full_msg: str, count: int) -> UiFailureTotal:
+ def from_pair(cls, full_msg: str, total: int) -> UiFailureTotal:
parts = full_msg.split(" ", 1)
if len(parts) > 1 and parts[0].isdigit():
error_code, msg = parts
- return cls(full_msg, count, int(error_code), msg)
- return cls(full_msg, count, None, full_msg)
+ return cls(full_msg, total, int(error_code), msg)
+ return cls(full_msg, total, None, full_msg)
def get_tasks_info_sorted(progress: Progress) -> tuple[list[TaskInfo], bool]:
diff --git a/cyberdrop_dl/utils/ffmpeg.py b/cyberdrop_dl/utils/ffmpeg.py
index e137abba9..52ecb8dca 100644
--- a/cyberdrop_dl/utils/ffmpeg.py
+++ b/cyberdrop_dl/utils/ffmpeg.py
@@ -242,7 +242,7 @@ class Stream:
return self.codec_name
@classmethod
- def validate(cls, stream_info: Mapping[str, Any]) -> dict[str, Any]:
+ def validate(cls, stream_info: StreamDict) -> dict[str, Any]:
info = get_valid_dict(cls, stream_info)
tags = Tags(CIMultiDict(stream_info.get("tags", {})))
duration: float | str | None = stream_info.get("duration") or tags.get("duration")
@@ -251,7 +251,7 @@ class Stream:
return info | {"tags": tags, "duration": duration, "bitrate": bitrate}
@classmethod
- def from_dict(cls, stream_info: Mapping[str, Any]) -> Self:
+ def from_dict(cls, stream_info: StreamDict) -> Self:
return cls(**cls.validate(stream_info))
def as_dict(self) -> dict[str, Any]:
diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.py
index 4d61792d4..5b607f2f2 100644
--- a/cyberdrop_dl/utils/logger.py
+++ b/cyberdrop_dl/utils/logger.py
@@ -108,7 +108,7 @@ class QueuedLogger:
class NoPaddingLogRender(LogRender):
cdl_padding = 0
- def __call__(
+ def __call__( # type: ignore[reportIncompatibleMethodOverride]
self,
console: Console,
renderables: Iterable[ConsoleRenderable],