Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix handling of None in allowed_domains.

Lukas Anzinger9d9dea0d6970
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 232e96cbb..36f809699 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -54,12 +54,16 @@ class OffsiteMiddleware(object):
if not allowed_domains:
return re.compile('') # allow all by default
url_pattern = re.compile("^https?://.*$")
+ domains = []
for domain in allowed_domains:
- if url_pattern.match(domain):
+ if domain is None:
+ continue
+ elif url_pattern.match(domain):
message = ("allowed_domains accepts only domains, not URLs. "
"Ignoring URL entry %s in allowed_domains." % domain)
warnings.warn(message, URLWarning)
- domains = [re.escape(d) for d in allowed_domains if d is not None]
+ else:
+ domains.append(re.escape(domain))
regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
return re.compile(regex)
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 7511aa568..51c328943 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -55,21 +55,21 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
- def _get_spider(self):
- return Spider('foo')
+ def _get_spiderargs(self):
+ return dict(name='foo')
class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
- def _get_spider(self):
- bad_hostname = urlparse('http:////scrapytest.org').hostname
- return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])
+ def _get_spiderargs(self):
+ bad_hostname = urlparse('http:////scrapytest.org').hostname
+ return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])
def test_process_spider_output(self):
- res = Response('http://scrapytest.org')
- reqs = [Request('http://scrapytest.org/1')]
- out = list(self.mw.process_spider_output(res, reqs, self.spider))
- self.assertEqual(out, reqs)
+ res = Response('http://scrapytest.org')
+ reqs = [Request('http://scrapytest.org/1')]
+ out = list(self.mw.process_spider_output(res, reqs, self.spider))
+ self.assertEqual(out, reqs)
class TestOffsiteMiddleware5(TestOffsiteMiddleware4):