Scrapy, a fast high-level web crawling & scraping framework for Python.
Typing for build_from_*. (#6326)
Andrey Rakhmatullind7da298e0637
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 2db085081..93a0c51bc 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -366,7 +366,8 @@ class ExecutionEngine:self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)self.spider = spiderif hasattr(scheduler, "open"):- yield scheduler.open(spider)+ if d := scheduler.open(spider):+ yield dyield self.scraper.open_spider(spider)assert self.crawler.statsself.crawler.stats.open_spider(spider)diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex f41b83a67..b2209e53f 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -322,6 +322,7 @@ class Scheduler(BaseScheduler):def _mq(self):"""Create a new priority queue instance, with in-memory storage"""+ assert self.crawlerreturn build_from_crawler(self.pqclass,self.crawler,@@ -331,6 +332,7 @@ class Scheduler(BaseScheduler):def _dq(self):"""Create a new priority queue instance, with disk storage"""+ assert self.crawlerassert self.dqdirstate = self._read_dqs_state(self.dqdir)q = build_from_crawler(diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 1db9ace28..ccfe78891 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -445,7 +445,9 @@ class CrawlerProcess(CrawlerRunner):d.addBoth(self._stop_reactor)resolver_class = load_object(self.settings["DNS_RESOLVER"])- resolver = build_from_crawler(resolver_class, self, reactor=reactor)+ # We pass self, which is CrawlerProcess, instead of Crawler here,+ # which works because the default resolvers only use crawler.settings.+ resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type]resolver.install_on_reactor()tp = reactor.getThreadPool()tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 7f83d06fb..faf52e44a 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -1,5 +1,7 @@"""Helper functions which don't fit anywhere else"""+from __future__ import annotations+import astimport hashlibimport inspect@@ -22,6 +24,8 @@ from typing import (Iterable,List,Optional,+ Type,+ TypeVar,Union,cast,)@@ -32,9 +36,11 @@ from scrapy.utils.datatypes import LocalWeakReferencedCacheif TYPE_CHECKING:from scrapy import Spider-+ from scrapy.crawler import Crawler+ from scrapy.settings import BaseSettings_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes+T = TypeVar("T")def arg_to_iter(arg: Any) -> Iterable[Any]:@@ -177,7 +183,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):return instance-def build_from_crawler(objcls, crawler, /, *args, **kwargs):+def build_from_crawler(+ objcls: Type[T], crawler: Crawler, /, *args: Any, **kwargs: Any+) -> T:"""Construct a class instance using its ``from_crawler`` constructor.``*args`` and ``**kwargs`` are forwarded to the constructor.@@ -185,20 +193,22 @@ def build_from_crawler(objcls, crawler, /, *args, **kwargs):Raises ``TypeError`` if the resulting instance is ``None``."""if hasattr(objcls, "from_crawler"):- instance = objcls.from_crawler(crawler, *args, **kwargs)+ instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined]method_name = "from_crawler"elif hasattr(objcls, "from_settings"):- instance = objcls.from_settings(crawler.settings, *args, **kwargs)+ instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined]method_name = "from_settings"else:instance = objcls(*args, **kwargs)method_name = "__new__"if instance is None:raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")- return instance+ return cast(T, instance)-def build_from_settings(objcls, settings, /, *args, **kwargs):+def build_from_settings(+ objcls: Type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any+) -> T:"""Construct a class instance using its ``from_settings`` constructor.``*args`` and ``**kwargs`` are forwarded to the constructor.@@ -206,14 +216,14 @@ def build_from_settings(objcls, settings, /, *args, **kwargs):Raises ``TypeError`` if the resulting instance is ``None``."""if hasattr(objcls, "from_settings"):- instance = objcls.from_settings(settings, *args, **kwargs)+ instance = objcls.from_settings(settings, *args, **kwargs) # type: ignore[attr-defined]method_name = "from_settings"else:instance = objcls(*args, **kwargs)method_name = "__new__"if instance is None:raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")- return instance+ return cast(T, instance)@contextmanager@@ -290,7 +300,7 @@ def is_generator_with_return_value(callable: Callable) -> bool:return bool(_generator_callbacks_cache[callable])-def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None:+def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None:"""Logs a warning if a callable is a generator function and includesa 'return' statement with a value different than None