Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix Flake8 errors
OmarFarrag40e0a11aa8dd
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex c1ca0ca67..f1b101780 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -7,18 +7,16 @@ See documentation in docs/topics/feed-exports.rstimport osimport sysimport logging-import posixpathfrom tempfile import NamedTemporaryFilefrom datetime import datetimefrom urllib.parse import urlparse, unquote-from ftplib import FTPfrom zope.interface import Interface, implementerfrom twisted.internet import defer, threadsfrom w3lib.url import file_uri_to_pathfrom scrapy import signals-from scrapy.utils.ftp import ftp_makedirs_cwd, ftp_store_file+from scrapy.utils.ftp import ftp_store_filefrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.log import failure_to_exc_info@@ -175,7 +173,7 @@ class FTPFeedStorage(BlockingFeedStorage):def _store_in_thread(self, file):ftp_store_file(path=self.path, file=file, host=self.host,- port=self.port, username=self.username,+ port=self.port, username=self.username,password=self.password, use_active_mode=self.use_active_mode)diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 2d286eed8..7e9b12c0e 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -28,7 +28,7 @@ from scrapy.utils.python import to_bytesfrom scrapy.utils.request import referer_strfrom scrapy.utils.boto import is_botocorefrom scrapy.utils.datatypes import CaselessDict-from scrapy.utils.ftp import ftp_makedirs_cwd, ftp_store_file+from scrapy.utils.ftp import ftp_store_filelogger = logging.getLogger(__name__)@@ -265,25 +265,25 @@ class FTPFilesStore(object):FTP_USERNAME = NoneFTP_PASSWORD = NoneUSE_ACTIVE_MODE = None-+def __init__(self, uri):assert uri.startswith('ftp://')- u = urlparse(uri)+ u = urlparse(uri)self.port = u.portself.host = u.hostnameself.port = int(u.port or 21)self.username = u.username or self.FTP_USERNAMEself.password = u.password or self.FTP_PASSWORDself.basedir = u.path.rstrip('/')-- def persist_file(self, path, buf, info, meta=None, headers=None):++ def persist_file(self, path, buf, info, meta=None, headers=None):path = '%s/%s' % (self.basedir, path)return threads.deferToThread(ftp_store_file, path=path, file=buf,host=self.host, port=self.port, username=self.username,password=self.password, use_active_mode=self.USE_ACTIVE_MODE)-+def stat_file(self, path, info):def _stat_file(path):try:@@ -298,8 +298,8 @@ class FTPFilesStore(object):ftp.retrbinary('RETR %s' % file_path, m.update)return {'last_modified': last_modified, 'checksum': m.hexdigest()}# The file doesn't exist- except Exception as e :- return {}+ except Exception:+ return {}return threads.deferToThread(_stat_file, path)@@ -381,7 +381,7 @@ class FilesPipeline(MediaPipeline):ftp_store.FTP_USERNAME = settings['FTP_USER']ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE')-+store_uri = settings['FILES_STORE']return cls(store_uri, settings=settings)diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.pyindex 1bb754a69..f07bdd748 100644--- a/scrapy/utils/ftp.py+++ b/scrapy/utils/ftp.py@@ -17,7 +17,8 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):if first_call:ftp.cwd(path)-def ftp_store_file(++def ftp_store_file(*, path, file, host, port,username, password, use_active_mode=False):"""Opens a FTP connection with passed credentials,sets current directorydiff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 65d24314e..61f2d059d 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -66,8 +66,9 @@ def get_gcs_content_and_delete(bucket, path):return content, acl, blob-def get_ftp_content_and_delete(path, host ,port,- username, password, use_active_mode=False):+def get_ftp_content_and_delete(+ path, host, port,username,+ password, use_active_mode=False):from ftplib import FTPftp = FTP()ftp.connect(host, port)@@ -75,6 +76,7 @@ def get_ftp_content_and_delete(path, host ,port,if use_active_mode:ftp.set_pasv(False)ftp_data = []+def buffer_data(data):ftp_data.append(data)ftp.retrbinary('RETR %s' % path, buffer_data)@@ -82,7 +84,7 @@ def get_ftp_content_and_delete(path, host ,port,ftp.cwd(dirname)ftp.delete(filename)return "".join(ftp_data)-+def get_crawler(spidercls=None, settings_dict=None):"""Return an unconfigured Crawler object. If settings_dict is given, itdiff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex fc0453a97..e5bad2ed0 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -367,6 +367,7 @@ class TestGCSFilesStore(unittest.TestCase):self.assertEqual(blob.content_type, 'application/octet-stream')self.assertIn(expected_policy, acl)+class TestFTPFileStore(unittest.TestCase):@defer.inlineCallbacksdef test_persist(self):@@ -386,10 +387,12 @@ class TestFTPFileStore(unittest.TestCase):self.assertIn('checksum', stat)self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6')path = '%s/%s' % (store.basedir, path)- content = get_ftp_content_and_delete(path, store.host, store.port,+ content = get_ftp_content_and_delete(+ path, store.host, store.port,store.username, store.password, store.USE_ACTIVE_MODE)self.assertEqual(data.decode(), content)+class ItemWithFiles(Item):file_urls = Field()files = Field()