Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Update for Python 3.7+

Adrián Chaves3e994bda45ac
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 7580011ac..3c36ef002 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -205,7 +205,7 @@ BaseItemExporter
['field1', 'field2']
- - A dict [3]_ where keys are fields and values are output names::
+ - A dict where keys are fields and values are output names::
{'field1': 'Field 1', 'field2': 'Field 2'}
@@ -214,14 +214,6 @@ BaseItemExporter
all their possible fields, exporters that do not support exporting
a different subset of fields per item will only export the fields
found in the first item exported.
- .. [3] Dicts preserve insertion order since `Python 3.7`_
- (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version
- of Python, use an OrderedDict_ to enforce a specific field order.
-
- .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html
- .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation
- .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html
- .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict
.. attribute:: export_empty_fields
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index ad12f26d6..76cbe4d4b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -2,13 +2,13 @@
Item Exporters are used to export/serialize items into different formats.
"""
-from collections import Mapping
import csv
import io
import marshal
import pickle
import pprint
import warnings
+from collections.abc import Mapping
from xml.sax.saxutils import XMLGenerator
from itemadapter import is_item, ItemAdapter
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 2bbe38481..6cacc63e1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,6 +1,5 @@
import json
import copy
-from collections import OrderedDict
from collections.abc import MutableMapping
from importlib import import_module
from pprint import pformat
@@ -199,7 +198,7 @@ class BaseSettings(MutableMapping):
return dict(value)
def getdictorlist(self, name, default=None):
- """Get a setting value as either an ``OrderedDict`` or a list.
+ """Get a setting value as either a :class:`dict` or a :class:`list`.
If the setting is already a dict or a list, a copy of it will be
returned.
@@ -209,7 +208,7 @@ class BaseSettings(MutableMapping):
For example, settings populated from the command line will return:
- - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to
+ - ``{'key1': 'value1', 'key2': 'value2'}`` if set to
``'{"key1": "value1", "key2": "value2"}'``
- ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
@@ -222,10 +221,10 @@ class BaseSettings(MutableMapping):
"""
value = self.get(name, default)
if value is None:
- return OrderedDict()
+ return {}
if isinstance(value, str):
try:
- return json.loads(value, object_pairs_hook=OrderedDict)
+ return json.loads(value)
except ValueError:
return value.split(',')
return copy.deepcopy(value)
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 6ba7428f6..096cd3116 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -4,7 +4,6 @@ import marshal
import pickle
import tempfile
import unittest
-from collections import OrderedDict
from io import BytesIO
from datetime import datetime
from warnings import catch_warnings, filterwarnings
@@ -114,11 +113,11 @@ class BaseItemExporterTest(unittest.TestCase):
self.assertEqual(name, 'John\xa3')
ie = self._get_exporter(
- fields_to_export=OrderedDict([('name', u'名稱')])
+ fields_to_export={'name': '名稱'}
)
self.assertEqual(
list(ie._get_serialized_fields(self.i)),
- [(u'名稱', u'John\xa3')]
+ [('名稱', 'John\xa3')]
)
def test_field_custom_serializer(self):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 83aabbdc7..9098e035d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -11,7 +11,7 @@ import sys
import tempfile
import warnings
from abc import ABC, abstractmethod
-from collections import defaultdict, OrderedDict
+from collections import defaultdict
from contextlib import ExitStack
from io import BytesIO
from logging import getLogger
@@ -998,9 +998,7 @@ class FeedExportTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_export_items_field_names(self):
items = [{'foo': 'bar'}]
- header = OrderedDict((
- ("foo", "Foo"),
- ))
+ header = {'foo': 'Foo'}
rows = [{'Foo': 'bar'}]
settings = {'FEED_EXPORT_FIELDS': header}
yield self.assertExported(items, list(header.values()), rows,
@@ -1023,9 +1021,7 @@ class FeedExportTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_export_items_json_field_names(self):
items = [{'foo': 'bar'}]
- header = OrderedDict((
- ("foo", "Foo"),
- ))
+ header = {'foo': 'Foo'}
rows = [{'Foo': 'bar'}]
settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
yield self.assertExported(items, list(header.values()), rows,