Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Misc typing improvements. (#6494)

Andrey Rakhmatullinf65e64a7243d
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index c7946605b..1ce155da7 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -13,8 +13,7 @@ from scrapy.utils.misc import load_object, set_environ
class TextTestResult(_TextTestResult):
def printSummary(self, start: float, stop: float) -> None:
write = self.stream.write
- # _WritelnDecorator isn't implemented in typeshed yet
- writeln = self.stream.writeln # type: ignore[attr-defined]
+ writeln = self.stream.writeln
run = self.testsRun
plural = "s" if run != 1 else ""
@@ -84,7 +83,7 @@ class Command(ScrapyCommand):
with set_environ(SCRAPY_CHECK="true"):
for spidername in args or spider_loader.list():
spidercls = spider_loader.load(spidername)
- spidercls.start_requests = lambda s: conman.from_spider(s, result)
+ spidercls.start_requests = lambda s: conman.from_spider(s, result) # type: ignore[assignment,method-assign,return-value]
tested_methods = conman.tested_methods_from_spidercls(spidercls)
if opts.list:
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index a9b7a6eee..2ac281212 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -4,7 +4,7 @@ import shutil
import string
from importlib import import_module
from pathlib import Path
-from typing import Optional, Union, cast
+from typing import Any, Optional, Union, cast
from urllib.parse import urlparse
import scrapy
@@ -122,7 +122,7 @@ class Command(ScrapyCommand):
name: str,
url: str,
template_name: str,
- ):
+ ) -> dict[str, Any]:
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
return {
"project_name": self.settings.get("BOT_NAME"),
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index bd1fad14b..ff2bb8ab9 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -38,9 +38,10 @@ _T = TypeVar("_T")
class Command(BaseRunSpiderCommand):
requires_project = True
- spider = None
+ spider: Optional[Spider] = None
items: dict[int, list[Any]] = {}
requests: dict[int, list[Request]] = {}
+ spidercls: Optional[type[Spider]]
first_response = None
@@ -261,10 +262,11 @@ class Command(BaseRunSpiderCommand):
yield self.prepare_request(spider, Request(url), opts)
if self.spidercls:
- self.spidercls.start_requests = _start_requests
+ self.spidercls.start_requests = _start_requests # type: ignore[assignment,method-assign]
def start_parsing(self, url: str, opts: argparse.Namespace) -> None:
assert self.crawler_process
+ assert self.spidercls
self.crawler_process.crawl(self.spidercls, **opts.spargs)
self.pcrawler = list(self.crawler_process.crawlers)[0]
self.crawler_process.start()
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index bb09d066f..f3d74eccf 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -100,7 +100,7 @@ class ExecutionEngine:
)
downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"])
self.downloader: Downloader = downloader_cls(crawler)
- self.scraper = Scraper(crawler)
+ self.scraper: Scraper = Scraper(crawler)
self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = (
spider_closed_callback
)
@@ -325,7 +325,7 @@ class ExecutionEngine:
raise RuntimeError(f"No open spider to crawl: {request}")
d: Deferred[Union[Response, Request]] = self._download(request)
# Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type
- d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type]
+ d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[call-overload]
return d2
def _downloaded(
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 29d7cb0c8..71a0d6aeb 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -55,7 +55,7 @@ class Slot:
MIN_RESPONSE_SIZE = 1024
def __init__(self, max_active_size: int = 5000000):
- self.max_active_size = max_active_size
+ self.max_active_size: int = max_active_size
self.queue: deque[QueueTuple] = deque()
self.active: set[Request] = set()
self.active_size: int = 0
@@ -316,7 +316,9 @@ class Scraper:
)
return None
- def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]:
+ def start_itemproc(
+ self, item: Any, *, response: Optional[Response]
+ ) -> Deferred[Any]:
"""Send *item* to the item pipelines for processing.
*response* is the source of the item data. If the item does not come
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 223e4192e..3c8513042 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -72,7 +72,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _process_spider_input(
self,
- scrape_func: ScrapeFunc,
+ scrape_func: ScrapeFunc[_T],
response: Response,
request: Request,
spider: Spider,
@@ -306,7 +306,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
def scrape_response(
self,
- scrape_func: ScrapeFunc,
+ scrape_func: ScrapeFunc[_T],
response: Response,
request: Request,
spider: Spider,
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index b0a4932e1..e75ef52ac 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -42,8 +42,9 @@ from scrapy.utils.reactor import (
)
if TYPE_CHECKING:
- from collections.abc import Generator
+ from collections.abc import Generator, Iterable
+ from scrapy.spiderloader import SpiderLoader
from scrapy.utils.request import RequestFingerprinter
@@ -178,16 +179,18 @@ class Crawler:
yield maybeDeferred(self.engine.stop)
@staticmethod
- def _get_component(component_class, components):
+ def _get_component(
+ component_class: type[_T], components: Iterable[Any]
+ ) -> Optional[_T]:
for component in components:
if isinstance(component, component_class):
return component
return None
- def get_addon(self, cls):
+ def get_addon(self, cls: type[_T]) -> Optional[_T]:
return self._get_component(cls, self.addons.addons)
- def get_downloader_middleware(self, cls):
+ def get_downloader_middleware(self, cls: type[_T]) -> Optional[_T]:
if not self.engine:
raise RuntimeError(
"Crawler.get_downloader_middleware() can only be called after "
@@ -195,7 +198,7 @@ class Crawler:
)
return self._get_component(cls, self.engine.downloader.middleware.middlewares)
- def get_extension(self, cls):
+ def get_extension(self, cls: type[_T]) -> Optional[_T]:
if not self.extensions:
raise RuntimeError(
"Crawler.get_extension() can only be called after the "
@@ -203,7 +206,7 @@ class Crawler:
)
return self._get_component(cls, self.extensions.middlewares)
- def get_item_pipeline(self, cls):
+ def get_item_pipeline(self, cls: type[_T]) -> Optional[_T]:
if not self.engine:
raise RuntimeError(
"Crawler.get_item_pipeline() can only be called after the "
@@ -211,7 +214,7 @@ class Crawler:
)
return self._get_component(cls, self.engine.scraper.itemproc.middlewares)
- def get_spider_middleware(self, cls):
+ def get_spider_middleware(self, cls: type[_T]) -> Optional[_T]:
if not self.engine:
raise RuntimeError(
"Crawler.get_spider_middleware() can only be called after the "
@@ -240,18 +243,18 @@ class CrawlerRunner:
)
@staticmethod
- def _get_spider_loader(settings: BaseSettings):
+ def _get_spider_loader(settings: BaseSettings) -> SpiderLoader:
"""Get SpiderLoader instance from settings"""
cls_path = settings.get("SPIDER_LOADER_CLASS")
loader_cls = load_object(cls_path)
verifyClass(ISpiderLoader, loader_cls)
- return loader_cls.from_settings(settings.frozencopy())
+ return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy()))
def __init__(self, settings: Union[dict[str, Any], Settings, None] = None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
- self.settings = settings
- self.spider_loader = self._get_spider_loader(settings)
+ self.settings: Settings = settings
+ self.spider_loader: SpiderLoader = self._get_spider_loader(settings)
self._crawlers: set[Crawler] = set()
self._active: set[Deferred[None]] = set()
self.bootstrap_failed = False
@@ -329,8 +332,7 @@ class CrawlerRunner:
def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler:
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
- # temporary cast until self.spider_loader is typed
- return Crawler(cast(type[Spider], spidercls), self.settings)
+ return Crawler(spidercls, self.settings)
def stop(self) -> Deferred[Any]:
"""
@@ -384,7 +386,7 @@ class CrawlerProcess(CrawlerRunner):
super().__init__(settings)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
- self._initialized_reactor = False
+ self._initialized_reactor: bool = False
def _signal_shutdown(self, signum: int, _: Any) -> None:
from twisted.internet import reactor
@@ -413,9 +415,7 @@ class CrawlerProcess(CrawlerRunner):
init_reactor = not self._initialized_reactor
self._initialized_reactor = True
# temporary cast until self.spider_loader is typed
- return Crawler(
- cast(type[Spider], spidercls), self.settings, init_reactor=init_reactor
- )
+ return Crawler(spidercls, self.settings, init_reactor=init_reactor)
def start(
self, stop_after_crawl: bool = True, install_signal_handlers: bool = True
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index b0cede97d..d913ca25d 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -88,7 +88,7 @@ class HttpCompressionMiddleware:
crawler.signals.connect(mw.open_spider, signals.spider_opened)
return mw
- def open_spider(self, spider):
+ def open_spider(self, spider: Spider) -> None:
if hasattr(spider, "download_maxsize"):
self._max_size = spider.download_maxsize
if hasattr(spider, "download_warnsize"):
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 421c58e68..81ba009d6 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -67,7 +67,7 @@ class RobotsTxtMiddleware:
if request.url.startswith("data:") or request.url.startswith("file:"):
return None
d: Deferred[Optional[RobotParser]] = maybeDeferred(
- self.robot_parser, request, spider # type: ignore[arg-type]
+ self.robot_parser, request, spider # type: ignore[call-overload]
)
d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider)
return d2
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index b1001dabb..7bfcbe6f3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -578,7 +578,7 @@ class FeedExporter:
return None
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
- d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type]
+ d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[call-overload]
d.addCallback(
self._handle_store_success, logmsg, spider, type(slot.storage).__name__
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 9314856c1..32e9ffe7c 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -550,7 +550,7 @@ class FilesPipeline(MediaPipeline):
path = self.file_path(request, info=info, item=item)
# maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
- dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type]
+ dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload]
dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess)
dfd2.addErrback(lambda _: None)
dfd2.addErrback(
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 3a0dee8f1..aeacadb1c 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -305,7 +305,11 @@ def process_parallel(
dfds, fireOnOneErrback=True, consumeErrors=True
)
d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r])
- d2.addErrback(lambda f: f.value.subFailure)
+
+ def eb(failure: Failure) -> Failure:
+ return failure.value.subFailure
+
+ d2.addErrback(eb)
return d2
diff --git a/tox.ini b/tox.ini
index dad15c6ab..79f72a0f2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -46,12 +46,12 @@ install_command =
[testenv:typing]
basepython = python3
deps =
- mypy==1.11.1
+ mypy==1.12.0
typing-extensions==4.12.2
- types-lxml==2024.8.7
+ types-lxml==2024.9.16
types-Pygments==2.18.0.20240506
- botocore-stubs==1.34.158
- boto3-stubs[s3]==1.34.158
+ botocore-stubs==1.35.39
+ boto3-stubs[s3]==1.35.39
attrs >= 18.2.0
Pillow >= 10.3.0
pyOpenSSL >= 24.2.1