Scrapy, a fast high-level web crawling & scraping framework for Python.
Update for Python 3.7+
Adrián Chaves3e994bda45ac
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rstindex 7580011ac..3c36ef002 100644--- a/docs/topics/exporters.rst+++ b/docs/topics/exporters.rst@@ -205,7 +205,7 @@ BaseItemExporter['field1', 'field2']- - A dict [3]_ where keys are fields and values are output names::+ - A dict where keys are fields and values are output names::{'field1': 'Field 1', 'field2': 'Field 2'}@@ -214,14 +214,6 @@ BaseItemExporterall their possible fields, exporters that do not support exportinga different subset of fields per item will only export the fieldsfound in the first item exported.- .. [3] Dicts preserve insertion order since `Python 3.7`_- (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version- of Python, use an OrderedDict_ to enforce a specific field order.-- .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html- .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation- .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html- .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict.. attribute:: export_empty_fieldsdiff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex ad12f26d6..76cbe4d4b 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -2,13 +2,13 @@Item Exporters are used to export/serialize items into different formats."""-from collections import Mappingimport csvimport ioimport marshalimport pickleimport pprintimport warnings+from collections.abc import Mappingfrom xml.sax.saxutils import XMLGeneratorfrom itemadapter import is_item, ItemAdapterdiff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex 2bbe38481..6cacc63e1 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -1,6 +1,5 @@import jsonimport copy-from collections import OrderedDictfrom collections.abc import MutableMappingfrom importlib import import_modulefrom pprint import pformat@@ -199,7 +198,7 @@ class BaseSettings(MutableMapping):return dict(value)def getdictorlist(self, name, default=None):- """Get a setting value as either an ``OrderedDict`` or a list.+ """Get a setting value as either a :class:`dict` or a :class:`list`.If the setting is already a dict or a list, a copy of it will bereturned.@@ -209,7 +208,7 @@ class BaseSettings(MutableMapping):For example, settings populated from the command line will return:- - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to+ - ``{'key1': 'value1', 'key2': 'value2'}`` if set to``'{"key1": "value1", "key2": "value2"}'``- ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``@@ -222,10 +221,10 @@ class BaseSettings(MutableMapping):"""value = self.get(name, default)if value is None:- return OrderedDict()+ return {}if isinstance(value, str):try:- return json.loads(value, object_pairs_hook=OrderedDict)+ return json.loads(value)except ValueError:return value.split(',')return copy.deepcopy(value)diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 6ba7428f6..096cd3116 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -4,7 +4,6 @@ import marshalimport pickleimport tempfileimport unittest-from collections import OrderedDictfrom io import BytesIOfrom datetime import datetimefrom warnings import catch_warnings, filterwarnings@@ -114,11 +113,11 @@ class BaseItemExporterTest(unittest.TestCase):self.assertEqual(name, 'John\xa3')ie = self._get_exporter(- fields_to_export=OrderedDict([('name', u'名稱')])+ fields_to_export={'name': '名稱'})self.assertEqual(list(ie._get_serialized_fields(self.i)),- [(u'名稱', u'John\xa3')]+ [('名稱', 'John\xa3')])def test_field_custom_serializer(self):diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 83aabbdc7..9098e035d 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -11,7 +11,7 @@ import sysimport tempfileimport warningsfrom abc import ABC, abstractmethod-from collections import defaultdict, OrderedDict+from collections import defaultdictfrom contextlib import ExitStackfrom io import BytesIOfrom logging import getLogger@@ -998,9 +998,7 @@ class FeedExportTest(FeedExportTestBase):@defer.inlineCallbacksdef test_export_items_field_names(self):items = [{'foo': 'bar'}]- header = OrderedDict((- ("foo", "Foo"),- ))+ header = {'foo': 'Foo'}rows = [{'Foo': 'bar'}]settings = {'FEED_EXPORT_FIELDS': header}yield self.assertExported(items, list(header.values()), rows,@@ -1023,9 +1021,7 @@ class FeedExportTest(FeedExportTestBase):@defer.inlineCallbacksdef test_export_items_json_field_names(self):items = [{'foo': 'bar'}]- header = OrderedDict((- ("foo", "Foo"),- ))+ header = {'foo': 'Foo'}rows = [{'Foo': 'bar'}]settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}yield self.assertExported(items, list(header.values()), rows,