Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Typing for build_from_*. (#6326)

Andrey Rakhmatullind7da298e0637
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 2db085081..93a0c51bc 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -366,7 +366,8 @@ class ExecutionEngine:
self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
self.spider = spider
if hasattr(scheduler, "open"):
- yield scheduler.open(spider)
+ if d := scheduler.open(spider):
+ yield d
yield self.scraper.open_spider(spider)
assert self.crawler.stats
self.crawler.stats.open_spider(spider)
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index f41b83a67..b2209e53f 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -322,6 +322,7 @@ class Scheduler(BaseScheduler):
def _mq(self):
"""Create a new priority queue instance, with in-memory storage"""
+ assert self.crawler
return build_from_crawler(
self.pqclass,
self.crawler,
@@ -331,6 +332,7 @@ class Scheduler(BaseScheduler):
def _dq(self):
"""Create a new priority queue instance, with disk storage"""
+ assert self.crawler
assert self.dqdir
state = self._read_dqs_state(self.dqdir)
q = build_from_crawler(
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 1db9ace28..ccfe78891 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -445,7 +445,9 @@ class CrawlerProcess(CrawlerRunner):
d.addBoth(self._stop_reactor)
resolver_class = load_object(self.settings["DNS_RESOLVER"])
- resolver = build_from_crawler(resolver_class, self, reactor=reactor)
+ # We pass self, which is CrawlerProcess, instead of Crawler here,
+ # which works because the default resolvers only use crawler.settings.
+ resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type]
resolver.install_on_reactor()
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 7f83d06fb..faf52e44a 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -1,5 +1,7 @@
"""Helper functions which don't fit anywhere else"""
+from __future__ import annotations
+
import ast
import hashlib
import inspect
@@ -22,6 +24,8 @@ from typing import (
Iterable,
List,
Optional,
+ Type,
+ TypeVar,
Union,
cast,
)
@@ -32,9 +36,11 @@ from scrapy.utils.datatypes import LocalWeakReferencedCache
if TYPE_CHECKING:
from scrapy import Spider
-
+ from scrapy.crawler import Crawler
+ from scrapy.settings import BaseSettings
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
+T = TypeVar("T")
def arg_to_iter(arg: Any) -> Iterable[Any]:
@@ -177,7 +183,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
return instance
-def build_from_crawler(objcls, crawler, /, *args, **kwargs):
+def build_from_crawler(
+ objcls: Type[T], crawler: Crawler, /, *args: Any, **kwargs: Any
+) -> T:
"""Construct a class instance using its ``from_crawler`` constructor.
``*args`` and ``**kwargs`` are forwarded to the constructor.
@@ -185,20 +193,22 @@ def build_from_crawler(objcls, crawler, /, *args, **kwargs):
Raises ``TypeError`` if the resulting instance is ``None``.
"""
if hasattr(objcls, "from_crawler"):
- instance = objcls.from_crawler(crawler, *args, **kwargs)
+ instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined]
method_name = "from_crawler"
elif hasattr(objcls, "from_settings"):
- instance = objcls.from_settings(crawler.settings, *args, **kwargs)
+ instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined]
method_name = "from_settings"
else:
instance = objcls(*args, **kwargs)
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
- return instance
+ return cast(T, instance)
-def build_from_settings(objcls, settings, /, *args, **kwargs):
+def build_from_settings(
+ objcls: Type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any
+) -> T:
"""Construct a class instance using its ``from_settings`` constructor.
``*args`` and ``**kwargs`` are forwarded to the constructor.
@@ -206,14 +216,14 @@ def build_from_settings(objcls, settings, /, *args, **kwargs):
Raises ``TypeError`` if the resulting instance is ``None``.
"""
if hasattr(objcls, "from_settings"):
- instance = objcls.from_settings(settings, *args, **kwargs)
+ instance = objcls.from_settings(settings, *args, **kwargs) # type: ignore[attr-defined]
method_name = "from_settings"
else:
instance = objcls(*args, **kwargs)
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
- return instance
+ return cast(T, instance)
@contextmanager
@@ -290,7 +300,7 @@ def is_generator_with_return_value(callable: Callable) -> bool:
return bool(_generator_callbacks_cache[callable])
-def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None:
+def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None:
"""
Logs a warning if a callable is a generator function and includes
a 'return' statement with a value different than None