Scrapy, a fast high-level web crawling & scraping framework for Python.
add ftp to the scheme list
nctl1444c05441450bc
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.pyindex 2d7115cc5..cda6ddc7e 100644--- a/scrapy/linkextractors/__init__.py+++ b/scrapy/linkextractors/__init__.py@@ -41,7 +41,8 @@ IGNORED_EXTENSIONS = [_re_type = type(re.compile("", 0))_matches = lambda url, regexs: any(r.search(url) for r in regexs)-_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'}+_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \+ 'file', 'ftp'}class FilteringLinkExtractor(object):diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 1d7c4f311..903032b52 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -451,6 +451,17 @@ class Base:Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),])+ def test_ftp_links(self):+ body = b"""+ <html><body>+ <div><a href="ftp://www.external.com/">An Item</a></div>+ </body></html>"""+ response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')+ lx = self.extractor_cls()+ self.assertEqual(lx.extract_links(response), [+ Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),+ ])+class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):extractor_cls = LxmlLinkExtractor@@ -471,4 +482,3 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):@pytest.mark.xfaildef test_restrict_xpaths_with_html_entities(self):super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()-