Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into ftp_files#3928
OmarFarragfc98aa6b671c
diff --cc scrapy/pipelines/files.pyindex 5383b05fe,6d55c8980..2d286eed8--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@@ -5,20 -5,14 +5,16 @@@ See documentation in topics/media-pipel"""import functoolsimport hashlib+ import logging+ import mimetypesimport os- import os.pathimport time- import logging+ from collections import defaultdictfrom email.utils import parsedate_tz, mktime_tz+from ftplib import FTP+ from io import BytesIO+from six.moves.urllib.parse import urlparse- from collections import defaultdict- import six-- try:- from cStringIO import StringIO as BytesIO- except ImportError:- from io import BytesIO+ from urllib.parse import urlparsefrom twisted.internet import defer, threads@@@ -32,8 -26,8 +28,9 @@@ from scrapy.utils.python import to_bytefrom scrapy.utils.request import referer_strfrom scrapy.utils.boto import is_botocorefrom scrapy.utils.datatypes import CaselessDict+from scrapy.utils.ftp import ftp_makedirs_cwd, ftp_store_file+logger = logging.getLogger(__name__)diff --cc scrapy/utils/ftp.pyindex 9992a916e,91d2439a9..1bb754a69--- a/scrapy/utils/ftp.py+++ b/scrapy/utils/ftp.py@@@ -1,8 -1,7 +1,9 @@@-from ftplib import error_perm+import posixpath++from ftplib import error_perm, FTPfrom posixpath import dirname+def ftp_makedirs_cwd(ftp, path, first_call=True):"""Set the current directory of the FTP connection given in the ``ftp``argument (as a ftplib.FTP object), creating all parent directories if theydiff --cc scrapy/utils/test.pyindex 59467f105,0f4cf8091..fd8f41137--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@@ -1,10 -1,8 +1,11 @@@"""This module contains some assorted functions used in tests"""++ import asyncio+ import os+from __future__ import absolute_import+from posixpath import split- import osfrom importlib import import_modulefrom twisted.trial.unittest import SkipTest@@@ -62,23 -62,7 +65,25 @@@ def get_gcs_content_and_delete(bucket,bucket.delete_blob(path)return content, acl, blob++def get_ftp_content_and_delete(path, host ,port,+ username, password, use_active_mode=False):+ from ftplib import FTP+ ftp = FTP()+ ftp.connect(host, port)+ ftp.login(username, password)+ if use_active_mode:+ ftp.set_pasv(False)+ ftp_data = []+ def buffer_data(data):+ ftp_data.append(data)+ ftp.retrbinary('RETR %s' % path, buffer_data)+ dirname, filename = split(path)+ ftp.cwd(dirname)+ ftp.delete(filename)+ return "".join(ftp_data)+++def get_crawler(spidercls=None, settings_dict=None):"""Return an unconfigured Crawler object. If settings_dict is given, itwill be used to populate the crawler settings with a project leveldiff --cc tests/test_pipeline_files.pyindex 000c1e2e2,141141671..fc0453a97--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@@ -15,14 -14,10 +14,11 @@@ from scrapy.pipelines.files import Filefrom scrapy.item import Item, Fieldfrom scrapy.http import Request, Responsefrom scrapy.settings import Settings- from scrapy.utils.python import to_bytesfrom scrapy.utils.test import assert_aws_environ, get_s3_content_and_deletefrom scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete+from scrapy.utils.test import get_ftp_content_and_deletefrom scrapy.utils.boto import is_botocore- from tests import mock-def _mocked_download_func(request, info):response = request.meta.get('response')