Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Prevent even more DeprecationWarnings

Eugenio Lacuestaeced544d64ac
diff --git a/scrapy/item.py b/scrapy/item.py
index 031b80a2d..393302dce 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -4,16 +4,22 @@ Scrapy Item
See documentation in docs/topics/item.rst
"""
+from abc import ABCMeta
from pprint import pformat
-from collections import MutableMapping
from copy import deepcopy
+import collections
-from abc import ABCMeta
import six
from scrapy.utils.trackref import object_ref
+if six.PY3:
+ MutableMapping = collections.abc.MutableMapping
+else:
+ MutableMapping = collections.MutableMapping
+
+
class BaseItem(object_ref):
"""Base class for all scraped items."""
pass
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index eb373c501..af39e887f 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -6,13 +6,20 @@ This module must not depend on any module outside the Standard Library.
"""
import copy
-import six
+import collections
import warnings
-from collections import OrderedDict, Mapping
+
+import six
from scrapy.exceptions import ScrapyDeprecationWarning
+if six.PY3:
+ Mapping = collections.abc.Mapping
+else:
+ Mapping = collections.Mapping
+
+
class MultiValueDictKeyError(KeyError):
def __init__(self, *args, **kwargs):
warnings.warn(
@@ -289,7 +296,7 @@ class MergeDict(object):
return self.__copy__()
-class LocalCache(OrderedDict):
+class LocalCache(collections.OrderedDict):
"""Dictionary with a finite number of keys.
Older items expires first.
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 60306eacb..2b3548bdd 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
-import re
from twisted.internet import reactor, error
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
from twisted.python import failure
@@ -31,18 +30,16 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
def _get_successful_crawler(self):
crawler = self.crawler
crawler.settings.set('ROBOTSTXT_OBEY', True)
- ROBOTS = re.sub(b'^\s+(?m)', b'', u'''
- User-Agent: *
- Disallow: /admin/
- Disallow: /static/
-
- # taken from https://en.wikipedia.org/robots.txt
- Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
- Disallow: /wiki/Käyttäjä:
-
- User-Agent: UnicödeBöt
- Disallow: /some/randome/page.html
- '''.encode('utf-8'))
+ ROBOTS = u"""
+User-Agent: *
+Disallow: /admin/
+Disallow: /static/
+# taken from https://en.wikipedia.org/robots.txt
+Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
+Disallow: /wiki/Käyttäjä:
+User-Agent: UnicödeBöt
+Disallow: /some/randome/page.html
+""".encode('utf-8')
response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
def return_response(request, spider):
deferred = Deferred()