Scrapy, a fast high-level web crawling & scraping framework for Python.
Additional typing for scraper and a small code change.
Andrey Rakhmatullin77bff0db0a6b
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex c760a4155..adbf8ef3d 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -3,11 +3,10 @@ extracts information from them"""import loggingfrom collections import deque-from collections.abc import Iterable-from typing import Union+from typing import Union, Optional, Tuple, Set, Deque, Any, Iterablefrom itemadapter import is_item-from twisted.internet import defer+from twisted.internet.defer import Deferred, inlineCallbacksfrom twisted.python.failure import Failurefrom scrapy import signals, Spider@@ -20,6 +19,9 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_valuefrom scrapy.utils.spider import iterate_spider_output+QUEUE_TUPLE = Tuple[Union[Response, Failure], Request, Deferred]++logger = logging.getLogger(__name__)@@ -28,46 +30,46 @@ class Slot:MIN_RESPONSE_SIZE = 1024- def __init__(self, max_active_size=5000000):+ def __init__(self, max_active_size: int = 5000000):self.max_active_size = max_active_size- self.queue = deque()- self.active = set()- self.active_size = 0- self.itemproc_size = 0- self.closing = None-- def add_response_request(self, response, request):- deferred = defer.Deferred()- self.queue.append((response, request, deferred))- if isinstance(response, Response):- self.active_size += max(len(response.body), self.MIN_RESPONSE_SIZE)+ self.queue: Deque[QUEUE_TUPLE] = deque()+ self.active: Set[Request] = set()+ self.active_size: int = 0+ self.itemproc_size: int = 0+ self.closing: Optional[Deferred] = None++ def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred:+ deferred = Deferred()+ self.queue.append((result, request, deferred))+ if isinstance(result, Response):+ self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)else:self.active_size += self.MIN_RESPONSE_SIZEreturn deferred- def next_response_request_deferred(self):+ def next_response_request_deferred(self) -> QUEUE_TUPLE:response, request, deferred = self.queue.popleft()self.active.add(request)return response, request, deferred- def finish_response(self, response, request):+ def finish_response(self, result: Union[Response, Failure], request: Request) -> None:self.active.remove(request)- if isinstance(response, Response):- self.active_size -= max(len(response.body), self.MIN_RESPONSE_SIZE)+ if isinstance(result, Response):+ self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE)else:self.active_size -= self.MIN_RESPONSE_SIZE- def is_idle(self):+ def is_idle(self) -> bool:return not (self.queue or self.active)- def needs_backout(self):+ def needs_backout(self) -> bool:return self.active_size > self.max_active_sizeclass Scraper:def __init__(self, crawler):- self.slot = None+ self.slot: Optional[Slot] = Noneself.spidermw = SpiderMiddlewareManager.from_crawler(crawler)itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])self.itemproc = itemproc_cls.from_crawler(crawler)@@ -76,32 +78,37 @@ class Scraper:self.signals = crawler.signalsself.logformatter = crawler.logformatter- @defer.inlineCallbacks- def open_spider(self, spider):+ @inlineCallbacks+ def open_spider(self, spider: Spider):"""Open the given spider for scraping and allocate resources for it"""self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE'))yield self.itemproc.open_spider(spider)- def close_spider(self, spider):+ def close_spider(self, spider: Spider) -> Deferred:"""Close a spider being scraped and release its resources"""- self.slot.closing = defer.Deferred()+ if self.slot is None:+ raise RuntimeError("Scraper slot not assigned")+ self.slot.closing = Deferred()self.slot.closing.addCallback(self.itemproc.close_spider)self._check_if_closing(spider)return self.slot.closing- def is_idle(self):+ def is_idle(self) -> bool:"""Return True if there isn't any more spiders to process"""return not self.slot- def _check_if_closing(self, spider):+ def _check_if_closing(self, spider: Spider) -> None:+ assert self.slot is not None # typingif self.slot.closing and self.slot.is_idle():self.slot.closing.callback(spider)- def enqueue_scrape(self, response, request, spider):- dfd = self.slot.add_response_request(response, request)+ def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:+ if self.slot is None:+ raise RuntimeError("Scraper slot not assigned")+ dfd = self.slot.add_response_request(result, request)def finish_scraping(_):- self.slot.finish_response(response, request)+ self.slot.finish_response(result, request)self._check_if_closing(spider)self._scrape_next(spider)return _@@ -115,12 +122,13 @@ class Scraper:self._scrape_next(spider)return dfd- def _scrape_next(self, spider):+ def _scrape_next(self, spider: Spider) -> None:+ assert self.slot is not None # typingwhile self.slot.queue:response, request, deferred = self.slot.next_response_request_deferred()self._scrape(response, request, spider).chainDeferred(deferred)- def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider):+ def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:"""Handle the downloaded response or failure through the spider callback/errback"""@@ -131,7 +139,7 @@ class Scraper:dfd.addCallback(self.handle_spider_output, request, result, spider)return dfd- def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider):+ def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:"""Handle the different cases of request's result been a Response or a Failure"""@@ -141,7 +149,7 @@ class Scraper:dfd = self.call_spider(result, request, spider)return dfd.addErrback(self._log_download_errors, result, request, spider)- def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider):+ def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:if isinstance(result, Response):if getattr(result, "request", None) is None:result.request = request@@ -156,7 +164,7 @@ class Scraper:dfd.addErrback(request.errback)return dfd.addCallback(iterate_spider_output)- def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider):+ def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None:exc = _failure.valueif isinstance(exc, CloseSpider):self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')@@ -177,7 +185,7 @@ class Scraper:spider=spider)- def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider):+ def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:if not result:return defer_succeed(None)it = iter_errback(result, self.handle_spider_error, request, response, spider)@@ -185,10 +193,12 @@ class Scraper:request, response, spider)return dfd- def _process_spidermw_output(self, output, request, response, spider):+ def _process_spidermw_output(self, output: Any, request: Request, response: Response,+ spider: Spider) -> Optional[Deferred]:"""Process each Request/Item (given in the output parameter) returnedfrom the given spider"""+ assert self.slot is not None # typingif isinstance(output, Request):self.crawler.engine.crawl(request=output, spider=spider)elif is_item(output):@@ -205,12 +215,18 @@ class Scraper:{'request': request, 'typename': typename},extra={'spider': spider},)+ return None- def _log_download_errors(self, spider_failure, download_failure, request, spider):+ def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request,+ spider: Spider) -> Union[Failure, None]:"""Log and silence errors that come from the engine (typically download- errors that got propagated thru here)+ errors that got propagated thru here).++ spider_failure: the value passed into the errback of self.call_spider()+ download_failure: the value passed into _scrape2() from+ ExecutionEngine._handle_downloader_output() as "result""""- if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest):+ if not download_failure.check(IgnoreRequest):if download_failure.frames:logkws = self.logformatter.download_error(download_failure, request, spider)logger.log(@@ -230,10 +246,12 @@ class Scraper:if spider_failure is not download_failure:return spider_failure+ return None- def _itemproc_finished(self, output, item, response, spider):+ def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None:"""ItemProcessor finished for the given ``item`` and returned ``output``"""+ assert self.slot is not None # typingself.slot.itemproc_size -= 1if isinstance(output, Failure):ex = output.value