Scrapy, a fast high-level web crawling & scraping framework for Python.
Wrap scrapy check in environment
Matthijs Vosd27c2c68ba4f
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.pyindex b8a9ef989..b29587fa7 100644--- a/scrapy/commands/check.py+++ b/scrapy/commands/check.py@@ -6,7 +6,7 @@ from unittest import TextTestRunner, TextTestResult as _TextTestResultfrom scrapy.commands import ScrapyCommandfrom scrapy.contracts import ContractsManager-from scrapy.utils.misc import load_object+from scrapy.utils.misc import load_object, set_environfrom scrapy.utils.conf import build_component_list@@ -68,16 +68,17 @@ class Command(ScrapyCommand):spider_loader = self.crawler_process.spider_loader- for spidername in args or spider_loader.list():- spidercls = spider_loader.load(spidername)- spidercls.start_requests = lambda s: conman.from_spider(s, result)-- tested_methods = conman.tested_methods_from_spidercls(spidercls)- if opts.list:- for method in tested_methods:- contract_reqs[spidercls.name].append(method)- elif tested_methods:- self.crawler_process.crawl(spidercls)+ with set_environ(SCRAPY_CHECK=True):+ for spidername in args or spider_loader.list():+ spidercls = spider_loader.load(spidername)+ spidercls.start_requests = lambda s: conman.from_spider(s, result)++ tested_methods = conman.tested_methods_from_spidercls(spidercls)+ if opts.list:+ for method in tested_methods:+ contract_reqs[spidercls.name].append(method)+ elif tested_methods:+ self.crawler_process.crawl(spidercls)# start checksif opts.list:diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex ddaa7f7bf..7a2cd18ea 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -1,6 +1,8 @@"""Helper functions which don't fit anywhere else"""+import osimport reimport hashlib+from contextlib import contextmanagerfrom importlib import import_modulefrom pkgutil import iter_modules@@ -10,7 +12,6 @@ from w3lib.html import replace_entitiesfrom scrapy.utils.python import flatten, to_unicodefrom scrapy.item import BaseItem-_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes@@ -40,7 +41,7 @@ def load_object(path):except ValueError:raise ValueError("Error loading object '%s': not a full path" % path)- module, name = path[:dot], path[dot+1:]+ module, name = path[:dot], path[dot + 1:]mod = import_module(module)try:@@ -85,9 +86,9 @@ def extract_regex(regex, text, encoding='utf-8'):regex = re.compile(regex, re.UNICODE)try:- strings = [regex.search(text).group('extract')] # named group+ strings = [regex.search(text).group('extract')] # named groupexcept Exception:- strings = regex.findall(text) # full regex or numbered groups+ strings = regex.findall(text) # full regex or numbered groupsstrings = flatten(strings)if isinstance(text, six.text_type):@@ -142,3 +143,21 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):return objcls.from_settings(settings, *args, **kwargs)else:return objcls(*args, **kwargs)+++@contextmanager+def set_environ(**kwargs):+ """Temporarily set environment variables inside the context manager and+ fully restore previous environment afterwards+ """++ original_env = {k: os.environ.get(k) for k in kwargs}+ os.environ.update(kwargs)+ try:+ yield+ finally:+ for k, v in original_env:+ if v is None:+ del os.environ[k]+ else:+ os.environ[k] = v