Scrapy, a fast high-level web crawling & scraping framework for Python.
Credentials from settings-Support custom paths-Remove close conenction
OmarFarrag9b1587ed1bc7
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 6f66460b8..c22404799 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -251,19 +251,30 @@ class GCSFilesStore(object):class FTPFilesStore(object):++ FTP_USERNAME = None+ FTP_PASSWORD = Nonedef __init__(self, uri):assert uri.startswith('ftp://')u = urlparse(uri)self.ftp = FTP()self.ftp.connect(u.hostname, u.port or '21')- self.ftp.login(u.username, u.password)- self.basedir = u.path + '/'- ftp_makedirs_cwd(self.ftp, self.basedir+'full')+ username = u.username or FTP_USERNAME+ password = u.password or FTP_PASSWORD+ self.ftp.login(username, password)+ self.basedir = u.pathdef persist_file(self, path, buf, info, meta=None, headers=None):buf.seek(0)- filename = path.split('/')[1]+ # If the path is like 'x/y/z.ext' the 'x/y' is rel_path and+ # 'z.ext' is file name+ # If path is only the file name 'z.ext', then rel_path is+ # the empty string and filename is 'z.ext'+ x = path.rsplit('/',1)+ rel_path, filename = ('/' + x[0], x[1]) if len(x) > 1 else ('', x[0])+ abs_path = self.basedir + rel_path+ ftp_makedirs_cwd(self.ftp, abs_path)return threads.deferToThread(self.ftp.storbinary,'STOR %s' % filename,@@ -281,9 +292,6 @@ class FTPFilesStore(object):except Exception as e :return {}return threads.deferToThread(_stat_file, path)-- def close_connection(self):- self.ftp.quit()class FilesPipeline(MediaPipeline):@@ -357,6 +365,10 @@ class FilesPipeline(MediaPipeline):gcs_store = cls.STORE_SCHEMES['gs']gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None++ ftp_store = cls.STORE_SCHEMES['ftp']+ ftp_store.FTP_USERNAME = settings['FTP_USER'] # Default is 'anonymous'+ ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD'] # Default is `guest`store_uri = settings['FILES_STORE']return cls(store_uri, settings=settings)@@ -497,11 +509,4 @@ class FilesPipeline(MediaPipeline):def file_path(self, request, response=None, info=None):media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()media_ext = os.path.splitext(request.url)[1]- return 'full/%s%s' % (media_guid, media_ext)-- def close_spider(self, spider):- try:- self.store.close_connection()- # If the store doesn't implement this function, pass- except AttributeError:- pass+ return 'full/%s%s' % (media_guid, media_ext)\ No newline at end of file