Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into ftp_files#3928

OmarFarragfc98aa6b671c
diff --cc scrapy/pipelines/files.py
index 5383b05fe,6d55c8980..2d286eed8
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@@ -5,20 -5,14 +5,16 @@@ See documentation in topics/media-pipel
"""
import functools
import hashlib
+ import logging
+ import mimetypes
import os
- import os.path
import time
- import logging
+ from collections import defaultdict
from email.utils import parsedate_tz, mktime_tz
+from ftplib import FTP
+ from io import BytesIO
+from six.moves.urllib.parse import urlparse
- from collections import defaultdict
- import six
-
- try:
- from cStringIO import StringIO as BytesIO
- except ImportError:
- from io import BytesIO
+ from urllib.parse import urlparse
from twisted.internet import defer, threads
@@@ -32,8 -26,8 +28,9 @@@ from scrapy.utils.python import to_byte
from scrapy.utils.request import referer_str
from scrapy.utils.boto import is_botocore
from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.ftp import ftp_makedirs_cwd, ftp_store_file
+
logger = logging.getLogger(__name__)
diff --cc scrapy/utils/ftp.py
index 9992a916e,91d2439a9..1bb754a69
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@@ -1,8 -1,7 +1,9 @@@
-from ftplib import error_perm
+import posixpath
+
+from ftplib import error_perm, FTP
from posixpath import dirname
+
def ftp_makedirs_cwd(ftp, path, first_call=True):
"""Set the current directory of the FTP connection given in the ``ftp``
argument (as a ftplib.FTP object), creating all parent directories if they
diff --cc scrapy/utils/test.py
index 59467f105,0f4cf8091..fd8f41137
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@@ -1,10 -1,8 +1,11 @@@
"""
This module contains some assorted functions used in tests
"""
+
+ import asyncio
+ import os
+from __future__ import absolute_import
+from posixpath import split
- import os
from importlib import import_module
from twisted.trial.unittest import SkipTest
@@@ -62,23 -62,7 +65,25 @@@ def get_gcs_content_and_delete(bucket,
bucket.delete_blob(path)
return content, acl, blob
+
+def get_ftp_content_and_delete(path, host ,port,
+ username, password, use_active_mode=False):
+ from ftplib import FTP
+ ftp = FTP()
+ ftp.connect(host, port)
+ ftp.login(username, password)
+ if use_active_mode:
+ ftp.set_pasv(False)
+ ftp_data = []
+ def buffer_data(data):
+ ftp_data.append(data)
+ ftp.retrbinary('RETR %s' % path, buffer_data)
+ dirname, filename = split(path)
+ ftp.cwd(dirname)
+ ftp.delete(filename)
+ return "".join(ftp_data)
+
++
def get_crawler(spidercls=None, settings_dict=None):
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
diff --cc tests/test_pipeline_files.py
index 000c1e2e2,141141671..fc0453a97
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@@ -15,14 -14,10 +14,11 @@@ from scrapy.pipelines.files import File
from scrapy.item import Item, Field
from scrapy.http import Request, Response
from scrapy.settings import Settings
- from scrapy.utils.python import to_bytes
from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
+from scrapy.utils.test import get_ftp_content_and_delete
from scrapy.utils.boto import is_botocore
- from tests import mock
-
def _mocked_download_func(request, info):
response = request.meta.get('response')