Scrapy, a fast high-level web crawling & scraping framework for Python.
Support Anonymous FTP
Paul Tremberth3ffa2c5759d0
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 93fa0a3a7..603c1e2e3 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:import refrom io import BytesIO-from six.moves.urllib.parse import urlparse, unquote+from six.moves.urllib.parse import unquotefrom twisted.internet import reactorfrom twisted.protocols.ftp import FTPClient, CommandFailed@@ -38,6 +38,8 @@ from twisted.internet.protocol import Protocol, ClientCreatorfrom scrapy.http import Responsefrom scrapy.responsetypes import responsetypes+from scrapy.utils.httpobj import urlparse_cached+class ReceivedDataProtocol(Protocol):def __init__(self, filename=None):@@ -64,14 +66,18 @@ class FTPDownloadHandler(object):"default": 503,}- def __init__(self, setting):- pass+ def __init__(self, settings):+ self.anonymous_user = settings['FTP_ANONYMOUS_USER']+ self.anonymous_password = settings['FTP_ANONYMOUS_PASSWORD']def download_request(self, request, spider):- parsed_url = urlparse(request.url)- creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"],- request.meta["ftp_password"],- passive=request.meta.get("ftp_passive", 1))+ parsed_url = urlparse_cached(request)+ user = request.meta.get("ftp_user", self.anonymous_user)+ password = request.meta.get("ftp_password")+ if user == self.anonymous_user and password is None:+ password = self.anonymous_password+ creator = ClientCreator(reactor, FTPClient, user, password,+ passive=request.meta.get("ftp_passive", 1))return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,request, unquote(parsed_url.path))diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex cb88bc2bf..a21c2584a 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -161,6 +161,9 @@ FEED_EXPORTERS_BASE = {FILES_STORE_S3_ACL = 'private'+FTP_ANONYMOUS_USER = 'anonymous'+FTP_ANONYMOUS_PASSWORD = '[email protected]'+HTTPCACHE_ENABLED = FalseHTTPCACHE_DIR = 'httpcache'HTTPCACHE_IGNORE_MISSING = Falsediff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 6333efceb..2d97573b3 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -676,10 +676,11 @@ class S3TestCase(unittest.TestCase):b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')-class FTPTestCase(unittest.TestCase):+class BaseFTPTestCase(unittest.TestCase):username = "scrapy"password = "passwd"+ req_meta = {"ftp_user": username, "ftp_password": password}if six.PY3:skip = "Twisted missing ftp support for PY3"@@ -722,7 +723,7 @@ class FTPTestCase(unittest.TestCase):def test_ftp_download_success(self):request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,- meta={"ftp_user": self.username, "ftp_password": self.password})+ meta=self.req_meta)d = self.download_handler.download_request(request, None)def _test(r):@@ -734,7 +735,7 @@ class FTPTestCase(unittest.TestCase):def test_ftp_download_path_with_spaces(self):request = Request(url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum,- meta={"ftp_user": self.username, "ftp_password": self.password}+ meta=self.req_meta)d = self.download_handler.download_request(request, None)@@ -746,7 +747,7 @@ class FTPTestCase(unittest.TestCase):def test_ftp_download_notexist(self):request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum,- meta={"ftp_user": self.username, "ftp_password": self.password})+ meta=self.req_meta)d = self.download_handler.download_request(request, None)def _test(r):@@ -755,8 +756,10 @@ class FTPTestCase(unittest.TestCase):def test_ftp_local_filename(self):local_fname = "/tmp/file.txt"+ meta = {"ftp_local_filename": local_fname}+ meta.update(self.req_meta)request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,- meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname})+ meta=meta)d = self.download_handler.download_request(request, None)def _test(r):@@ -768,13 +771,50 @@ class FTPTestCase(unittest.TestCase):os.remove(local_fname)return self._add_test_callbacks(d, _test)++class FTPTestCase(BaseFTPTestCase):+def test_invalid_credentials(self):from twisted.protocols.ftp import ConnectionLost+ meta = dict(self.req_meta)+ meta.update({"ftp_password": 'invalid'})request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,- meta={"ftp_user": self.username, "ftp_password": 'invalid'})+ meta=meta)d = self.download_handler.download_request(request, None)def _test(r):self.assertEqual(r.type, ConnectionLost)return self._add_test_callbacks(d, errback=_test)+++class AnonymousFTPTestCase(BaseFTPTestCase):++ username = "anonymous"+ req_meta = {}++ def setUp(self):+ from twisted.protocols.ftp import FTPRealm, FTPFactory+ from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler++ # setup dir and test file+ self.directory = self.mktemp()+ os.mkdir(self.directory)++ fp = FilePath(self.directory)+ fp.child('file.txt').setContent("I have the power!")+ fp.child('file with spaces.txt').setContent("Moooooooooo power!")++ # setup server for anonymous access+ realm = FTPRealm(anonymousRoot=self.directory)+ p = portal.Portal(realm)+ p.registerChecker(checkers.AllowAnonymousAccess(),+ credentials.IAnonymous)++ self.factory = FTPFactory(portal=p,+ userAnonymous=self.username)+ print("self.factory.allowAnonymous=%r" % self.factory.allowAnonymous)+ self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")+ self.portNum = self.port.getHost().port+ self.download_handler = FTPDownloadHandler(Settings())+ self.addCleanup(self.port.stopListening)