Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove six.moves occurrences from tests

Eugenio Lacuesta5ab0f189ce33
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 8aff7b525..15b1b24f5 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,6 +1,10 @@
-import sys, time, random, os, json
-from six.moves.urllib.parse import urlencode
+import json
+import os
+import random
+import sys
+import time
from subprocess import Popen, PIPE
+from urllib.parse import urlencode
from OpenSSL import SSL
from twisted.web.server import Site, NOT_DONE_YET
diff --git a/tests/spiders.py b/tests/spiders.py
index 7816bf7c7..72a428c50 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -3,7 +3,7 @@ Some spiders used for testing and benchmarking
"""
import time
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
from scrapy.spiders import Spider
from scrapy.http import Request
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3fc13eeb7..3307899b7 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -140,7 +140,7 @@ class CrawlTestCase(TestCase):
def test_unbounded_response(self):
# Completeness of responses without Content-Length or Transfer-Encoding
# can not be determined, we treat them as valid but flagged as "partial"
- from six.moves.urllib.parse import urlencode
+ from urllib.parse import urlencode
query = urlencode({'raw': '''\
HTTP/1.1 200 OK
Server: Apache-Coyote/1.1
diff --git a/tests/test_engine.py b/tests/test_engine.py
index d5b911a40..002c4e6bb 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -10,8 +10,10 @@ module with the ``runserver`` argument::
python test_engine.py runserver
"""
-import sys, os, re
-from six.moves.urllib.parse import urlparse
+import os
+import re
+import sys
+from urllib.parse import urlparse
from twisted.internet import reactor, defer
from twisted.web import server, static, util
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 7880c5bf8..f151a1285 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,11 +1,11 @@
import re
import json
import marshal
+import pickle
import tempfile
import unittest
from io import BytesIO
from datetime import datetime
-from six.moves import cPickle as pickle
import lxml.etree
import six
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index b13b16b41..7a71669c3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2,13 +2,13 @@ import os
import csv
import json
import warnings
-from io import BytesIO
import tempfile
import shutil
import string
+from io import BytesIO
from unittest import mock
-from six.moves.urllib.parse import urljoin, urlparse, quote
-from six.moves.urllib.request import pathname2url
+from urllib.parse import urljoin, urlparse, quote
+from urllib.request import pathname2url
from zope.interface.verify import verifyObject
from twisted.trial import unittest
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 0a9ed500a..45ddb42ba 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from unittest import TestCase
from scrapy.http import Request, Response
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 807265981..62d2847d7 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,12 +3,10 @@ import cgi
import unittest
import re
import json
-from unittest import mock
-from urllib.parse import unquote_to_bytes
+import xmlrpc.client as xmlrpclib
import warnings
-
-from six.moves import xmlrpc_client as xmlrpclib
-from six.moves.urllib.parse import urlparse, parse_qs, unquote
+from unittest import mock
+from urllib.parse import parse_qs, unquote, unquote_to_bytes, urlparse
from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
from scrapy.utils.python import to_bytes, to_unicode
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index bd40e4103..dede4bf12 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -4,9 +4,9 @@ import time
from tempfile import mkdtemp
from shutil import rmtree
from unittest import mock
-from six.moves.urllib.parse import urlparse
-from six import BytesIO
+from urllib.parse import urlparse
+from six import BytesIO
from twisted.trial import unittest
from twisted.internet import defer
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index ae1236bcb..2b62a378d 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -1,15 +1,15 @@
import json
import os
import time
-
-from six.moves.urllib.parse import urlsplit, urlunsplit
+from urllib.parse import urlsplit, urlunsplit
from threading import Thread
+
from libmproxy import controller, proxy
from netlib import http_auth
from testfixtures import LogCapture
-
from twisted.internet import defer
from twisted.trial.unittest import TestCase
+
from scrapy.utils.test import get_crawler
from scrapy.http import Request
from tests.spiders import SimpleSpider, SingleRequestSpider
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 7e4af0d4c..5833591af 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -1,13 +1,12 @@
from unittest import TestCase
-
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+import warnings
from scrapy.http import Response, Request
from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
-from scrapy.spidermiddlewares.offsite import URLWarning
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning
from scrapy.utils.test import get_crawler
-import warnings
+
class TestOffsiteMiddleware(TestCase):
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 21439c20e..23c38d9b1 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from unittest import TestCase
import warnings
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
index 22e39821c..bea0cf3e5 100644
--- a/tests/test_urlparse_monkeypatches.py
+++ b/tests/test_urlparse_monkeypatches.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
import unittest
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 47877f555..8782e4c53 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -192,14 +192,6 @@ class SequenceExcludeTest(unittest.TestCase):
self.assertIn(20, d)
self.assertNotIn(15, d)
- def test_six_range(self):
- import six.moves
- seq = six.moves.range(10**3, 10**6)
- d = SequenceExclude(seq)
- self.assertIn(10**2, d)
- self.assertIn(10**7, d)
- self.assertNotIn(10**4, d)
-
def test_range_step(self):
seq = range(10, 20, 3)
d = SequenceExclude(seq)
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 003bb9b02..0d8c46657 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -5,8 +5,6 @@ from twisted.python.failure import Failure
from scrapy.utils.defer import mustbe_deferred, process_chain, \
process_chain_both, process_parallel, iter_errback
-from six.moves import xrange
-
class MustbeDeferredTest(unittest.TestCase):
def test_success_function(self):
@@ -83,7 +81,7 @@ class IterErrbackTest(unittest.TestCase):
def test_iter_errback_good(self):
def itergood():
- for x in xrange(10):
+ for x in range(10):
yield x
errors = []
@@ -93,7 +91,7 @@ class IterErrbackTest(unittest.TestCase):
def test_iter_errback_bad(self):
def iterbad():
- for x in xrange(10):
+ for x in range(10):
if x == 5:
a = 1/0
yield x
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 4f9f7a370..cf8ad1f23 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -1,9 +1,10 @@
import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
+
class HttpobjUtilsTest(unittest.TestCase):
def test_urlparse_cached(self):
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index bea4dade3..6ebf290c0 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,12 +1,13 @@
import os
import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (response_httprepr, open_in_browser,
get_meta_refresh, get_base_url, response_status_message)
+
__doctests__ = ['scrapy.utils.response']
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index e6588055c..a8e37d7b8 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,14 +1,15 @@
# -*- coding: utf-8 -*-
import unittest
+from urllib.parse import urlparse
import six
-from six.moves.urllib.parse import urlparse
from scrapy.spiders import Spider
from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
add_http_if_no_scheme, guess_scheme,
parse_url, strip_url)
+
__doctests__ = ['scrapy.utils.url']