Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support Anonymous FTP

Paul Tremberth3ffa2c5759d0
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 93fa0a3a7..603c1e2e3 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
import re
from io import BytesIO
-from six.moves.urllib.parse import urlparse, unquote
+from six.moves.urllib.parse import unquote
from twisted.internet import reactor
from twisted.protocols.ftp import FTPClient, CommandFailed
@@ -38,6 +38,8 @@ from twisted.internet.protocol import Protocol, ClientCreator
from scrapy.http import Response
from scrapy.responsetypes import responsetypes
+from scrapy.utils.httpobj import urlparse_cached
+
class ReceivedDataProtocol(Protocol):
def __init__(self, filename=None):
@@ -64,14 +66,18 @@ class FTPDownloadHandler(object):
"default": 503,
}
- def __init__(self, setting):
- pass
+ def __init__(self, settings):
+ self.anonymous_user = settings['FTP_ANONYMOUS_USER']
+ self.anonymous_password = settings['FTP_ANONYMOUS_PASSWORD']
def download_request(self, request, spider):
- parsed_url = urlparse(request.url)
- creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"],
- request.meta["ftp_password"],
- passive=request.meta.get("ftp_passive", 1))
+ parsed_url = urlparse_cached(request)
+ user = request.meta.get("ftp_user", self.anonymous_user)
+ password = request.meta.get("ftp_password")
+ if user == self.anonymous_user and password is None:
+ password = self.anonymous_password
+ creator = ClientCreator(reactor, FTPClient, user, password,
+ passive=request.meta.get("ftp_passive", 1))
return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
request, unquote(parsed_url.path))
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index cb88bc2bf..a21c2584a 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -161,6 +161,9 @@ FEED_EXPORTERS_BASE = {
FILES_STORE_S3_ACL = 'private'
+FTP_ANONYMOUS_USER = 'anonymous'
+FTP_ANONYMOUS_PASSWORD = '[email protected]'
+
HTTPCACHE_ENABLED = False
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_IGNORE_MISSING = False
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 6333efceb..2d97573b3 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -676,10 +676,11 @@ class S3TestCase(unittest.TestCase):
b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
-class FTPTestCase(unittest.TestCase):
+class BaseFTPTestCase(unittest.TestCase):
username = "scrapy"
password = "passwd"
+ req_meta = {"ftp_user": username, "ftp_password": password}
if six.PY3:
skip = "Twisted missing ftp support for PY3"
@@ -722,7 +723,7 @@ class FTPTestCase(unittest.TestCase):
def test_ftp_download_success(self):
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
- meta={"ftp_user": self.username, "ftp_password": self.password})
+ meta=self.req_meta)
d = self.download_handler.download_request(request, None)
def _test(r):
@@ -734,7 +735,7 @@ class FTPTestCase(unittest.TestCase):
def test_ftp_download_path_with_spaces(self):
request = Request(
url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum,
- meta={"ftp_user": self.username, "ftp_password": self.password}
+ meta=self.req_meta
)
d = self.download_handler.download_request(request, None)
@@ -746,7 +747,7 @@ class FTPTestCase(unittest.TestCase):
def test_ftp_download_notexist(self):
request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum,
- meta={"ftp_user": self.username, "ftp_password": self.password})
+ meta=self.req_meta)
d = self.download_handler.download_request(request, None)
def _test(r):
@@ -755,8 +756,10 @@ class FTPTestCase(unittest.TestCase):
def test_ftp_local_filename(self):
local_fname = "/tmp/file.txt"
+ meta = {"ftp_local_filename": local_fname}
+ meta.update(self.req_meta)
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
- meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname})
+ meta=meta)
d = self.download_handler.download_request(request, None)
def _test(r):
@@ -768,13 +771,50 @@ class FTPTestCase(unittest.TestCase):
os.remove(local_fname)
return self._add_test_callbacks(d, _test)
+
+class FTPTestCase(BaseFTPTestCase):
+
def test_invalid_credentials(self):
from twisted.protocols.ftp import ConnectionLost
+ meta = dict(self.req_meta)
+ meta.update({"ftp_password": 'invalid'})
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
- meta={"ftp_user": self.username, "ftp_password": 'invalid'})
+ meta=meta)
d = self.download_handler.download_request(request, None)
def _test(r):
self.assertEqual(r.type, ConnectionLost)
return self._add_test_callbacks(d, errback=_test)
+
+
+class AnonymousFTPTestCase(BaseFTPTestCase):
+
+ username = "anonymous"
+ req_meta = {}
+
+ def setUp(self):
+ from twisted.protocols.ftp import FTPRealm, FTPFactory
+ from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
+
+ # setup dir and test file
+ self.directory = self.mktemp()
+ os.mkdir(self.directory)
+
+ fp = FilePath(self.directory)
+ fp.child('file.txt').setContent("I have the power!")
+ fp.child('file with spaces.txt').setContent("Moooooooooo power!")
+
+ # setup server for anonymous access
+ realm = FTPRealm(anonymousRoot=self.directory)
+ p = portal.Portal(realm)
+ p.registerChecker(checkers.AllowAnonymousAccess(),
+ credentials.IAnonymous)
+
+ self.factory = FTPFactory(portal=p,
+ userAnonymous=self.username)
+ print("self.factory.allowAnonymous=%r" % self.factory.allowAnonymous)
+ self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
+ self.portNum = self.port.getHost().port
+ self.download_handler = FTPDownloadHandler(Settings())
+ self.addCleanup(self.port.stopListening)