Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

test for files pipeline ftp store

OmarFarrag0e8770a2f4e9
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4b935c51b..59467f105 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -3,6 +3,7 @@ This module contains some assorted functions used in tests
"""
from __future__ import absolute_import
+from posixpath import split
import os
from importlib import import_module
@@ -61,6 +62,23 @@ def get_gcs_content_and_delete(bucket, path):
bucket.delete_blob(path)
return content, acl, blob
+def get_ftp_content_and_delete(path, host ,port,
+ username, password, use_active_mode=False):
+ from ftplib import FTP
+ ftp = FTP()
+ ftp.connect(host, port)
+ ftp.login(username, password)
+ if use_active_mode:
+ ftp.set_pasv(False)
+ ftp_data = []
+ def buffer_data(data):
+ ftp_data.append(data)
+ ftp.retrbinary('RETR %s' % path, buffer_data)
+ dirname, filename = split(path)
+ ftp.cwd(dirname)
+ ftp.delete(filename)
+ return "".join(ftp_data)
+
def get_crawler(spidercls=None, settings_dict=None):
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 0c5aaaa44..000c1e2e2 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -11,13 +11,14 @@ from six import BytesIO
from twisted.trial import unittest
from twisted.internet import defer
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore
+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore, FTPFilesStore
from scrapy.item import Item, Field
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.utils.python import to_bytes
from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
+from scrapy.utils.test import get_ftp_content_and_delete
from scrapy.utils.boto import is_botocore
from tests import mock
@@ -365,6 +366,28 @@ class TestGCSFilesStore(unittest.TestCase):
self.assertEqual(blob.content_type, 'application/octet-stream')
self.assertIn(expected_policy, acl)
+class TestFTPFileStore(unittest.TestCase):
+ @defer.inlineCallbacks
+ def test_persist(self):
+ uri = os.environ.get('FTP_TEST_FILE_URI')
+ if not uri:
+ raise unittest.SkipTest("No FTP URI available for testing")
+ data = b"TestFTPFilesStore: \xe2\x98\x83"
+ buf = BytesIO(data)
+ meta = {'foo': 'bar'}
+ path = 'full/filename'
+ store = FTPFilesStore(uri)
+ empty_dict = yield store.stat_file(path, info=None)
+ self.assertEqual(empty_dict, {})
+ yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
+ stat = yield store.stat_file(path, info=None)
+ self.assertIn('last_modified', stat)
+ self.assertIn('checksum', stat)
+ self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6')
+ path = '%s/%s' % (store.basedir, path)
+ content = get_ftp_content_and_delete(path, store.host, store.port,
+ store.username, store.password, store.USE_ACTIVE_MODE)
+ self.assertEqual(data.decode(), content)
class ItemWithFiles(Item):
file_urls = Field()