Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix Flake8 errors

OmarFarrag40e0a11aa8dd
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c1ca0ca67..f1b101780 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -7,18 +7,16 @@ See documentation in docs/topics/feed-exports.rst
import os
import sys
import logging
-import posixpath
from tempfile import NamedTemporaryFile
from datetime import datetime
from urllib.parse import urlparse, unquote
-from ftplib import FTP
from zope.interface import Interface, implementer
from twisted.internet import defer, threads
from w3lib.url import file_uri_to_path
from scrapy import signals
-from scrapy.utils.ftp import ftp_makedirs_cwd, ftp_store_file
+from scrapy.utils.ftp import ftp_store_file
from scrapy.exceptions import NotConfigured
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.log import failure_to_exc_info
@@ -175,7 +173,7 @@ class FTPFeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file):
ftp_store_file(
path=self.path, file=file, host=self.host,
- port=self.port, username=self.username,
+ port=self.port, username=self.username,
password=self.password, use_active_mode=self.use_active_mode
)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 2d286eed8..7e9b12c0e 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -28,7 +28,7 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.request import referer_str
from scrapy.utils.boto import is_botocore
from scrapy.utils.datatypes import CaselessDict
-from scrapy.utils.ftp import ftp_makedirs_cwd, ftp_store_file
+from scrapy.utils.ftp import ftp_store_file
logger = logging.getLogger(__name__)
@@ -265,25 +265,25 @@ class FTPFilesStore(object):
FTP_USERNAME = None
FTP_PASSWORD = None
USE_ACTIVE_MODE = None
-
+
def __init__(self, uri):
assert uri.startswith('ftp://')
- u = urlparse(uri)
+ u = urlparse(uri)
self.port = u.port
self.host = u.hostname
self.port = int(u.port or 21)
self.username = u.username or self.FTP_USERNAME
self.password = u.password or self.FTP_PASSWORD
self.basedir = u.path.rstrip('/')
-
- def persist_file(self, path, buf, info, meta=None, headers=None):
+
+ def persist_file(self, path, buf, info, meta=None, headers=None):
path = '%s/%s' % (self.basedir, path)
return threads.deferToThread(
ftp_store_file, path=path, file=buf,
host=self.host, port=self.port, username=self.username,
password=self.password, use_active_mode=self.USE_ACTIVE_MODE
)
-
+
def stat_file(self, path, info):
def _stat_file(path):
try:
@@ -298,8 +298,8 @@ class FTPFilesStore(object):
ftp.retrbinary('RETR %s' % file_path, m.update)
return {'last_modified': last_modified, 'checksum': m.hexdigest()}
# The file doesn't exist
- except Exception as e :
- return {}
+ except Exception:
+ return {}
return threads.deferToThread(_stat_file, path)
@@ -381,7 +381,7 @@ class FilesPipeline(MediaPipeline):
ftp_store.FTP_USERNAME = settings['FTP_USER']
ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']
ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE')
-
+
store_uri = settings['FILES_STORE']
return cls(store_uri, settings=settings)
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index 1bb754a69..f07bdd748 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -17,7 +17,8 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
if first_call:
ftp.cwd(path)
-def ftp_store_file(
+
+def ftp_store_file(
*, path, file, host, port,
username, password, use_active_mode=False):
"""Opens a FTP connection with passed credentials,sets current directory
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 65d24314e..61f2d059d 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -66,8 +66,9 @@ def get_gcs_content_and_delete(bucket, path):
return content, acl, blob
-def get_ftp_content_and_delete(path, host ,port,
- username, password, use_active_mode=False):
+def get_ftp_content_and_delete(
+ path, host, port,username,
+ password, use_active_mode=False):
from ftplib import FTP
ftp = FTP()
ftp.connect(host, port)
@@ -75,6 +76,7 @@ def get_ftp_content_and_delete(path, host ,port,
if use_active_mode:
ftp.set_pasv(False)
ftp_data = []
+
def buffer_data(data):
ftp_data.append(data)
ftp.retrbinary('RETR %s' % path, buffer_data)
@@ -82,7 +84,7 @@ def get_ftp_content_and_delete(path, host ,port,
ftp.cwd(dirname)
ftp.delete(filename)
return "".join(ftp_data)
-
+
def get_crawler(spidercls=None, settings_dict=None):
"""Return an unconfigured Crawler object. If settings_dict is given, it
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index fc0453a97..e5bad2ed0 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -367,6 +367,7 @@ class TestGCSFilesStore(unittest.TestCase):
self.assertEqual(blob.content_type, 'application/octet-stream')
self.assertIn(expected_policy, acl)
+
class TestFTPFileStore(unittest.TestCase):
@defer.inlineCallbacks
def test_persist(self):
@@ -386,10 +387,12 @@ class TestFTPFileStore(unittest.TestCase):
self.assertIn('checksum', stat)
self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6')
path = '%s/%s' % (store.basedir, path)
- content = get_ftp_content_and_delete(path, store.host, store.port,
+ content = get_ftp_content_and_delete(
+ path, store.host, store.port,
store.username, store.password, store.USE_ACTIVE_MODE)
self.assertEqual(data.decode(), content)
+
class ItemWithFiles(Item):
file_urls = Field()
files = Field()