Scrapy, a fast high-level web crawling & scraping framework for Python.
Ignore a domain in allowed_domains with port and issue a warning (#4413)
Lukas Anzingerccc4d88779cf
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.pyindex 36f809699..2fab572e6 100644--- a/scrapy/spidermiddlewares/offsite.py+++ b/scrapy/spidermiddlewares/offsite.py@@ -53,7 +53,8 @@ class OffsiteMiddleware(object):allowed_domains = getattr(spider, 'allowed_domains', None)if not allowed_domains:return re.compile('') # allow all by default- url_pattern = re.compile("^https?://.*$")+ url_pattern = re.compile(r"^https?://.*$")+ port_pattern = re.compile(r":\d+$")domains = []for domain in allowed_domains:if domain is None:@@ -62,6 +63,10 @@ class OffsiteMiddleware(object):message = ("allowed_domains accepts only domains, not URLs. ""Ignoring URL entry %s in allowed_domains." % domain)warnings.warn(message, URLWarning)+ elif port_pattern.search(domain):+ message = ("allowed_domains accepts only domains without ports. "+ "Ignoring entry %s in allowed_domains." % domain)+ warnings.warn(message, PortWarning)else:domains.append(re.escape(domain))regex = r'^(.*\.)?(%s)$' % '|'.join(domains)@@ -74,3 +79,7 @@ class OffsiteMiddleware(object):class URLWarning(Warning):pass+++class PortWarning(Warning):+ passdiff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.pyindex 51c328943..b96807bc2 100644--- a/tests/test_spidermiddleware_offsite.py+++ b/tests/test_spidermiddleware_offsite.py@@ -4,7 +4,7 @@ import warningsfrom scrapy.http import Response, Requestfrom scrapy.spiders import Spider-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning, PortWarningfrom scrapy.utils.test import get_crawler@@ -26,7 +26,8 @@ class TestOffsiteMiddleware(TestCase):Request('http://scrapy.org/1'),Request('http://sub.scrapy.org/1'),Request('http://offsite.tld/letmepass', dont_filter=True),- Request('http://scrapy.test.org/')]+ Request('http://scrapy.test.org/'),+ Request('http://scrapy.test.org:8000/')]offsite_reqs = [Request('http://scrapy2.org'),Request('http://offsite.tld/'),Request('http://offsite.tld/scrapytest.org'),@@ -80,3 +81,13 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4):warnings.simplefilter("always")self.mw.get_host_regex(self.spider)assert issubclass(w[-1].category, URLWarning)+++class TestOffsiteMiddleware6(TestOffsiteMiddleware4):++ def test_get_host_regex(self):+ self.spider.allowed_domains = ['scrapytest.org:8000', 'scrapy.org', 'scrapy.test.org']+ with warnings.catch_warnings(record=True) as w:+ warnings.simplefilter("always")+ self.mw.get_host_regex(self.spider)+ assert issubclass(w[-1].category, PortWarning)