Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Make default referrer policy customizable via settings

Paul Tremberth842ce131aa66
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index a5931a3d5..15a134dd3 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -234,6 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True
+REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 44a599433..88041d8f0 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -8,6 +8,7 @@ from scrapy.http import Request
from scrapy.exceptions import NotConfigured
from scrapy.utils.python import to_native_str
from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.misc import load_object
LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
@@ -221,16 +222,27 @@ _policy_classes = {p.name: p for p in (
class RefererMiddleware(object):
- def __init__(self, policy_class=DefaultReferrerPolicy):
- self.default_policy = policy_class
+ def __init__(self, settings={}):
+ policy = settings.get('REFERER_POLICY')
+ if policy is not None:
+ try:
+ self.default_policy = load_object(policy)
+ except ValueError:
+ try:
+ self.default_policy = _policy_classes[policy]
+ except:
+ raise NotConfigured("Unknown referrer policy name %r" % policy)
+ else:
+ self.default_policy = DefaultReferrerPolicy
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('REFERER_ENABLED'):
raise NotConfigured
- return cls()
+ return cls(crawler.settings)
def policy(self, response, request):
+ # policy set in request's meta dict takes precedence over default policy
policy_name = request.meta.get('referrer_policy')
if policy_name is None:
policy_name = to_native_str(