Scrapy, a fast high-level web crawling & scraping framework for Python.
Revert "Default values for OffsiteMiddleware"
Eugenio Lacuesta60c2ef86f0c4
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.pyindex 3b7f194e4..310166cad 100644--- a/scrapy/spidermiddlewares/offsite.py+++ b/scrapy/spidermiddlewares/offsite.py@@ -19,9 +19,6 @@ class OffsiteMiddleware(object):def __init__(self, stats):self.stats = stats- # default values- self.host_regex = re.compile('') # allow all by default- self.domains_seen = set()@classmethoddef from_crawler(cls, crawler):@@ -55,7 +52,7 @@ class OffsiteMiddleware(object):"""Override this method to implement a different offsite policy"""allowed_domains = getattr(spider, 'allowed_domains', None)if not allowed_domains:- return+ return re.compile('') # allow all by defaulturl_pattern = re.compile("^https?://.*$")for domain in allowed_domains:if url_pattern.match(domain):@@ -65,9 +62,8 @@ class OffsiteMiddleware(object):return re.compile(regex)def spider_opened(self, spider):- host_regex = self.get_host_regex(spider)- if host_regex:- self.host_regex = host_regex+ self.host_regex = self.get_host_regex(spider)+ self.domains_seen = set()class URLWarning(Warning):