Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix handling of None in allowed_domains.
Lukas Anzinger9d9dea0d6970
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.pyindex 232e96cbb..36f809699 100644--- a/scrapy/spidermiddlewares/offsite.py+++ b/scrapy/spidermiddlewares/offsite.py@@ -54,12 +54,16 @@ class OffsiteMiddleware(object):if not allowed_domains:return re.compile('') # allow all by defaulturl_pattern = re.compile("^https?://.*$")+ domains = []for domain in allowed_domains:- if url_pattern.match(domain):+ if domain is None:+ continue+ elif url_pattern.match(domain):message = ("allowed_domains accepts only domains, not URLs. ""Ignoring URL entry %s in allowed_domains." % domain)warnings.warn(message, URLWarning)- domains = [re.escape(d) for d in allowed_domains if d is not None]+ else:+ domains.append(re.escape(domain))regex = r'^(.*\.)?(%s)$' % '|'.join(domains)return re.compile(regex)diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.pyindex 7511aa568..51c328943 100644--- a/tests/test_spidermiddleware_offsite.py+++ b/tests/test_spidermiddleware_offsite.py@@ -55,21 +55,21 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):class TestOffsiteMiddleware3(TestOffsiteMiddleware2):- def _get_spider(self):- return Spider('foo')+ def _get_spiderargs(self):+ return dict(name='foo')class TestOffsiteMiddleware4(TestOffsiteMiddleware3):- def _get_spider(self):- bad_hostname = urlparse('http:////scrapytest.org').hostname- return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])+ def _get_spiderargs(self):+ bad_hostname = urlparse('http:////scrapytest.org').hostname+ return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])def test_process_spider_output(self):- res = Response('http://scrapytest.org')- reqs = [Request('http://scrapytest.org/1')]- out = list(self.mw.process_spider_output(res, reqs, self.spider))- self.assertEqual(out, reqs)+ res = Response('http://scrapytest.org')+ reqs = [Request('http://scrapytest.org/1')]+ out = list(self.mw.process_spider_output(res, reqs, self.spider))+ self.assertEqual(out, reqs)class TestOffsiteMiddleware5(TestOffsiteMiddleware4):