Scrapy, a fast high-level web crawling & scraping framework for Python.
test for files pipeline ftp store
OmarFarrag0e8770a2f4e9
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 4b935c51b..59467f105 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -3,6 +3,7 @@ This module contains some assorted functions used in tests"""from __future__ import absolute_import+from posixpath import splitimport osfrom importlib import import_module@@ -61,6 +62,23 @@ def get_gcs_content_and_delete(bucket, path):bucket.delete_blob(path)return content, acl, blob+def get_ftp_content_and_delete(path, host ,port,+ username, password, use_active_mode=False):+ from ftplib import FTP+ ftp = FTP()+ ftp.connect(host, port)+ ftp.login(username, password)+ if use_active_mode:+ ftp.set_pasv(False)+ ftp_data = []+ def buffer_data(data):+ ftp_data.append(data)+ ftp.retrbinary('RETR %s' % path, buffer_data)+ dirname, filename = split(path)+ ftp.cwd(dirname)+ ftp.delete(filename)+ return "".join(ftp_data)+def get_crawler(spidercls=None, settings_dict=None):"""Return an unconfigured Crawler object. If settings_dict is given, itwill be used to populate the crawler settings with a project leveldiff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 0c5aaaa44..000c1e2e2 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -11,13 +11,14 @@ from six import BytesIOfrom twisted.trial import unittestfrom twisted.internet import defer-from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore, FTPFilesStorefrom scrapy.item import Item, Fieldfrom scrapy.http import Request, Responsefrom scrapy.settings import Settingsfrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import assert_aws_environ, get_s3_content_and_deletefrom scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete+from scrapy.utils.test import get_ftp_content_and_deletefrom scrapy.utils.boto import is_botocorefrom tests import mock@@ -365,6 +366,28 @@ class TestGCSFilesStore(unittest.TestCase):self.assertEqual(blob.content_type, 'application/octet-stream')self.assertIn(expected_policy, acl)+class TestFTPFileStore(unittest.TestCase):+ @defer.inlineCallbacks+ def test_persist(self):+ uri = os.environ.get('FTP_TEST_FILE_URI')+ if not uri:+ raise unittest.SkipTest("No FTP URI available for testing")+ data = b"TestFTPFilesStore: \xe2\x98\x83"+ buf = BytesIO(data)+ meta = {'foo': 'bar'}+ path = 'full/filename'+ store = FTPFilesStore(uri)+ empty_dict = yield store.stat_file(path, info=None)+ self.assertEqual(empty_dict, {})+ yield store.persist_file(path, buf, info=None, meta=meta, headers=None)+ stat = yield store.stat_file(path, info=None)+ self.assertIn('last_modified', stat)+ self.assertIn('checksum', stat)+ self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6')+ path = '%s/%s' % (store.basedir, path)+ content = get_ftp_content_and_delete(path, store.host, store.port,+ store.username, store.password, store.USE_ACTIVE_MODE)+ self.assertEqual(data.decode(), content)class ItemWithFiles(Item):file_urls = Field()