Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Ignore a domain in allowed_domains with port and issue a warning (#4413)

Lukas Anzingerccc4d88779cf
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 36f809699..2fab572e6 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -53,7 +53,8 @@ class OffsiteMiddleware(object):
allowed_domains = getattr(spider, 'allowed_domains', None)
if not allowed_domains:
return re.compile('') # allow all by default
- url_pattern = re.compile("^https?://.*$")
+ url_pattern = re.compile(r"^https?://.*$")
+ port_pattern = re.compile(r":\d+$")
domains = []
for domain in allowed_domains:
if domain is None:
@@ -62,6 +63,10 @@ class OffsiteMiddleware(object):
message = ("allowed_domains accepts only domains, not URLs. "
"Ignoring URL entry %s in allowed_domains." % domain)
warnings.warn(message, URLWarning)
+ elif port_pattern.search(domain):
+ message = ("allowed_domains accepts only domains without ports. "
+ "Ignoring entry %s in allowed_domains." % domain)
+ warnings.warn(message, PortWarning)
else:
domains.append(re.escape(domain))
regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
@@ -74,3 +79,7 @@ class OffsiteMiddleware(object):
class URLWarning(Warning):
pass
+
+
+class PortWarning(Warning):
+ pass
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 51c328943..b96807bc2 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -4,7 +4,7 @@ import warnings
from scrapy.http import Response, Request
from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning, PortWarning
from scrapy.utils.test import get_crawler
@@ -26,7 +26,8 @@ class TestOffsiteMiddleware(TestCase):
Request('http://scrapy.org/1'),
Request('http://sub.scrapy.org/1'),
Request('http://offsite.tld/letmepass', dont_filter=True),
- Request('http://scrapy.test.org/')]
+ Request('http://scrapy.test.org/'),
+ Request('http://scrapy.test.org:8000/')]
offsite_reqs = [Request('http://scrapy2.org'),
Request('http://offsite.tld/'),
Request('http://offsite.tld/scrapytest.org'),
@@ -80,3 +81,13 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
warnings.simplefilter("always")
self.mw.get_host_regex(self.spider)
assert issubclass(w[-1].category, URLWarning)
+
+
+class TestOffsiteMiddleware6(TestOffsiteMiddleware4):
+
+ def test_get_host_regex(self):
+ self.spider.allowed_domains = ['scrapytest.org:8000', 'scrapy.org', 'scrapy.test.org']
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter("always")
+ self.mw.get_host_regex(self.spider)
+ assert issubclass(w[-1].category, PortWarning)