Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Revert "Default values for OffsiteMiddleware"

Eugenio Lacuesta60c2ef86f0c4
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 3b7f194e4..310166cad 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -19,9 +19,6 @@ class OffsiteMiddleware(object):
def __init__(self, stats):
self.stats = stats
- # default values
- self.host_regex = re.compile('') # allow all by default
- self.domains_seen = set()
@classmethod
def from_crawler(cls, crawler):
@@ -55,7 +52,7 @@ class OffsiteMiddleware(object):
"""Override this method to implement a different offsite policy"""
allowed_domains = getattr(spider, 'allowed_domains', None)
if not allowed_domains:
- return
+ return re.compile('') # allow all by default
url_pattern = re.compile("^https?://.*$")
for domain in allowed_domains:
if url_pattern.match(domain):
@@ -65,9 +62,8 @@ class OffsiteMiddleware(object):
return re.compile(regex)
def spider_opened(self, spider):
- host_regex = self.get_host_regex(spider)
- if host_regex:
- self.host_regex = host_regex
+ self.host_regex = self.get_host_regex(spider)
+ self.domains_seen = set()
class URLWarning(Warning):