Scrapy, a fast high-level web crawling & scraping framework for Python.
Make default referrer policy customizable via settings
Paul Tremberth842ce131aa66
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex a5931a3d5..15a134dd3 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -234,6 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default settingREDIRECT_PRIORITY_ADJUST = +2REFERER_ENABLED = True+REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'RETRY_ENABLED = TrueRETRY_TIMES = 2 # initial response + 2 retries = 3 requestsdiff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 44a599433..88041d8f0 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -8,6 +8,7 @@ from scrapy.http import Requestfrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.python import to_native_strfrom scrapy.utils.httpobj import urlparse_cached+from scrapy.utils.misc import load_objectLOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)@@ -221,16 +222,27 @@ _policy_classes = {p.name: p for p in (class RefererMiddleware(object):- def __init__(self, policy_class=DefaultReferrerPolicy):- self.default_policy = policy_class+ def __init__(self, settings={}):+ policy = settings.get('REFERER_POLICY')+ if policy is not None:+ try:+ self.default_policy = load_object(policy)+ except ValueError:+ try:+ self.default_policy = _policy_classes[policy]+ except:+ raise NotConfigured("Unknown referrer policy name %r" % policy)+ else:+ self.default_policy = DefaultReferrerPolicy@classmethoddef from_crawler(cls, crawler):if not crawler.settings.getbool('REFERER_ENABLED'):raise NotConfigured- return cls()+ return cls(crawler.settings)def policy(self, response, request):+ # policy set in request's meta dict takes precedence over default policypolicy_name = request.meta.get('referrer_policy')if policy_name is None:policy_name = to_native_str(