Scrapy, a fast high-level web crawling & scraping framework for Python.
Warn or fail with exception on unknown policies
Paul Tremberth6916dd624094
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex bb184cc12..5ed40791b 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -3,6 +3,7 @@ RefererMiddleware: populates Request referer field, based on the Response whichoriginated it."""from six.moves.urllib.parse import urlparse+import warningsfrom w3lib.url import safe_url_string@@ -265,45 +266,69 @@ _policy_classes = {p.name: p for p in (DefaultReferrerPolicy,)}++def _load_policy_class(policy, warning_only=False):+ """+ Expect a string for the path to the policy class,+ otherwise try to interpret the string as a standard value+ from https://www.w3.org/TR/referrer-policy/#referrer-policies+ """+ try:+ return load_object(policy)+ except ValueError:+ try:+ return _policy_classes[policy.lower()]+ except KeyError:+ msg = "Could not load referrer policy %r" % policy+ if not warning_only:+ raise RuntimeError(msg)+ else:+ warnings.warn(msg, RuntimeWarning)+ return None++class RefererMiddleware(object):def __init__(self, settings=None):self.default_policy = DefaultReferrerPolicyif settings is not None:- policy = settings.get('REFERRER_POLICY')- if policy is not None:- # expect a string for the path to the policy class- try:- self.default_policy = load_object(policy)- except ValueError:- # otherwise try to interpret the string as standard- # https://www.w3.org/TR/referrer-policy/#referrer-policies- try:- self.default_policy = _policy_classes[policy.lower()]- except:- raise NotConfigured("Unknown referrer policy name %r" % policy)+ self.default_policy = _load_policy_class(+ settings.get('REFERRER_POLICY'))@classmethoddef from_crawler(cls, crawler):if not crawler.settings.getbool('REFERER_ENABLED'):raise NotConfiguredmw = cls(crawler.settings)++ # Note: this hook is a bit of a hack to intercept redirectionscrawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)+return mwdef policy(self, resp_or_url, request):"""Determine Referrer-Policy to use from a parent Response (or URL),and a Request to be sent.++ - if a valid policy is set in Request meta, it is used.+ - if the policy is set in meta but is wrong (e.g. a typo error),+ the policy from settings is used+ - if the policy is not set in Request meta,+ but there is a Referrer-policy header in the parent response,+ it is used if valid+ - otherwise, the policy from settings is used."""- # policy set in request's meta dict takes precedence over default policypolicy_name = request.meta.get('referrer_policy')if policy_name is None:if isinstance(resp_or_url, Response):policy_name = to_native_str(resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))- cls = _policy_classes.get(policy_name.lower()) if policy_name else self.default_policy- return cls()+ if policy_name is None:+ return self.default_policy()++ cls = _load_policy_class(policy_name, warning_only=True)+ return cls() if cls else self.default_policy()def process_spider_output(self, response, result, spider):def _set_referer(r):