Scrapy, a fast high-level web crawling & scraping framework for Python.
Full typing for scrapy/exporters.py. (#6275)
Andrey Rakhmatullin8985a04bd132
diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 79fd4e56f..fb4998099 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -3,16 +3,18 @@ Item Exporters are used to export/serialize items into different formats."""import csv-import ioimport marshalimport pickle # nosecimport pprint-from collections.abc import Mapping+from io import BytesIO, TextIOWrapper+from json import JSONEncoder+from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Unionfrom xml.sax.saxutils import XMLGenerator # nosec+from xml.sax.xmlreader import AttributesImpl # nosecfrom itemadapter import ItemAdapter, is_item-from scrapy.item import Item+from scrapy.item import Field, Itemfrom scrapy.utils.python import is_listlike, to_bytes, to_unicodefrom scrapy.utils.serialize import ScrapyJSONEncoder@@ -29,36 +31,42 @@ __all__ = [class BaseItemExporter:- def __init__(self, *, dont_fail=False, **kwargs):- self._kwargs = kwargs+ def __init__(self, *, dont_fail: bool = False, **kwargs: Any):+ self._kwargs: Dict[str, Any] = kwargsself._configure(kwargs, dont_fail=dont_fail)- def _configure(self, options, dont_fail=False):+ def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None:"""Configure the exporter by popping options from the ``options`` dict.If dont_fail is set, it won't raise an exception on unexpected options(useful for using with keyword arguments in subclasses ``__init__`` methods)"""- self.encoding = options.pop("encoding", None)- self.fields_to_export = options.pop("fields_to_export", None)- self.export_empty_fields = options.pop("export_empty_fields", False)- self.indent = options.pop("indent", None)+ self.encoding: Optional[str] = options.pop("encoding", None)+ self.fields_to_export: Union[Mapping[str, str], Iterable[str], None] = (+ options.pop("fields_to_export", None)+ )+ self.export_empty_fields: bool = options.pop("export_empty_fields", False)+ self.indent: Optional[int] = options.pop("indent", None)if not dont_fail and options:raise TypeError(f"Unexpected options: {', '.join(options.keys())}")- def export_item(self, item):+ def export_item(self, item: Any) -> None:raise NotImplementedError- def serialize_field(self, field, name, value):- serializer = field.get("serializer", lambda x: x)+ def serialize_field(+ self, field: Union[Mapping[str, Any], Field], name: str, value: Any+ ) -> Any:+ serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x)return serializer(value)- def start_exporting(self):+ def start_exporting(self) -> None:pass- def finish_exporting(self):+ def finish_exporting(self) -> None:pass- def _get_serialized_fields(self, item, default_value=None, include_empty=None):+ def _get_serialized_fields(+ self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None+ ) -> Iterable[Tuple[str, Any]]:"""Return the fields to export as an iterable of tuples(name, serialized_value)"""@@ -100,22 +108,22 @@ class BaseItemExporter:class JsonLinesItemExporter(BaseItemExporter):- def __init__(self, file, **kwargs):+ def __init__(self, file: BytesIO, **kwargs: Any):super().__init__(dont_fail=True, **kwargs)- self.file = file+ self.file: BytesIO = fileself._kwargs.setdefault("ensure_ascii", not self.encoding)- self.encoder = ScrapyJSONEncoder(**self._kwargs)+ self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs)- def export_item(self, item):+ def export_item(self, item: Any) -> None:itemdict = dict(self._get_serialized_fields(item))data = self.encoder.encode(itemdict) + "\n"self.file.write(to_bytes(data, self.encoding))class JsonItemExporter(BaseItemExporter):- def __init__(self, file, **kwargs):+ def __init__(self, file: BytesIO, **kwargs: Any):super().__init__(dont_fail=True, **kwargs)- self.file = file+ self.file: BytesIO = file# there is a small difference between the behaviour or JsonItemExporter.indent# and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent# the addition of newlines everywhere@@ -127,26 +135,26 @@ class JsonItemExporter(BaseItemExporter):self.encoder = ScrapyJSONEncoder(**self._kwargs)self.first_item = True- def _beautify_newline(self):+ def _beautify_newline(self) -> None:if self.indent is not None:self.file.write(b"\n")- def _add_comma_after_first(self):+ def _add_comma_after_first(self) -> None:if self.first_item:self.first_item = Falseelse:self.file.write(b",")self._beautify_newline()- def start_exporting(self):+ def start_exporting(self) -> None:self.file.write(b"[")self._beautify_newline()- def finish_exporting(self):+ def finish_exporting(self) -> None:self._beautify_newline()self.file.write(b"]")- def export_item(self, item):+ def export_item(self, item: Any) -> None:itemdict = dict(self._get_serialized_fields(item))data = to_bytes(self.encoder.encode(itemdict), self.encoding)self._add_comma_after_first()@@ -154,7 +162,7 @@ class JsonItemExporter(BaseItemExporter):class XmlItemExporter(BaseItemExporter):- def __init__(self, file, **kwargs):+ def __init__(self, file: BytesIO, **kwargs: Any):self.item_element = kwargs.pop("item_element", "item")self.root_element = kwargs.pop("root_element", "items")super().__init__(**kwargs)@@ -162,22 +170,22 @@ class XmlItemExporter(BaseItemExporter):self.encoding = "utf-8"self.xg = XMLGenerator(file, encoding=self.encoding)- def _beautify_newline(self, new_item=False):+ def _beautify_newline(self, new_item: bool = False) -> None:if self.indent is not None and (self.indent > 0 or new_item):self.xg.characters("\n")- def _beautify_indent(self, depth=1):+ def _beautify_indent(self, depth: int = 1) -> None:if self.indent:self.xg.characters(" " * self.indent * depth)- def start_exporting(self):+ def start_exporting(self) -> None:self.xg.startDocument()- self.xg.startElement(self.root_element, {})+ self.xg.startElement(self.root_element, AttributesImpl({}))self._beautify_newline(new_item=True)- def export_item(self, item):+ def export_item(self, item: Any) -> None:self._beautify_indent(depth=1)- self.xg.startElement(self.item_element, {})+ self.xg.startElement(self.item_element, AttributesImpl({}))self._beautify_newline()for name, value in self._get_serialized_fields(item, default_value=""):self._export_xml_field(name, value, depth=2)@@ -185,13 +193,13 @@ class XmlItemExporter(BaseItemExporter):self.xg.endElement(self.item_element)self._beautify_newline(new_item=True)- def finish_exporting(self):+ def finish_exporting(self) -> None:self.xg.endElement(self.root_element)self.xg.endDocument()- def _export_xml_field(self, name, serialized_value, depth):+ def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None:self._beautify_indent(depth=depth)- self.xg.startElement(name, {})+ self.xg.startElement(name, AttributesImpl({}))if hasattr(serialized_value, "items"):self._beautify_newline()for subname, value in serialized_value.items():@@ -213,17 +221,17 @@ class XmlItemExporter(BaseItemExporter):class CsvItemExporter(BaseItemExporter):def __init__(self,- file,- include_headers_line=True,- join_multivalued=",",- errors=None,- **kwargs,+ file: BytesIO,+ include_headers_line: bool = True,+ join_multivalued: str = ",",+ errors: Optional[str] = None,+ **kwargs: Any,):super().__init__(dont_fail=True, **kwargs)if not self.encoding:self.encoding = "utf-8"self.include_headers_line = include_headers_line- self.stream = io.TextIOWrapper(+ self.stream = TextIOWrapper(file,line_buffering=False,write_through=True,@@ -235,11 +243,13 @@ class CsvItemExporter(BaseItemExporter):self._headers_not_written = Trueself._join_multivalued = join_multivalued- def serialize_field(self, field, name, value):- serializer = field.get("serializer", self._join_if_needed)+ def serialize_field(+ self, field: Union[Mapping[str, Any], Field], name: str, value: Any+ ) -> Any:+ serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed)return serializer(value)- def _join_if_needed(self, value):+ def _join_if_needed(self, value: Any) -> Any:if isinstance(value, (list, tuple)):try:return self._join_multivalued.join(value)@@ -247,7 +257,7 @@ class CsvItemExporter(BaseItemExporter):passreturn value- def export_item(self, item):+ def export_item(self, item: Any) -> None:if self._headers_not_written:self._headers_not_written = Falseself._write_headers_and_set_fields_to_export(item)@@ -256,36 +266,38 @@ class CsvItemExporter(BaseItemExporter):values = list(self._build_row(x for _, x in fields))self.csv_writer.writerow(values)- def finish_exporting(self):+ def finish_exporting(self) -> None:self.stream.detach() # Avoid closing the wrapped file.- def _build_row(self, values):+ def _build_row(self, values: Iterable[Any]) -> Iterable[Any]:for s in values:try:yield to_unicode(s, self.encoding)except TypeError:yield s- def _write_headers_and_set_fields_to_export(self, item):+ def _write_headers_and_set_fields_to_export(self, item: Any) -> None:if self.include_headers_line:if not self.fields_to_export:# use declared field names, or keys if the item is a dictself.fields_to_export = ItemAdapter(item).field_names()+ fields: Iterable[str]if isinstance(self.fields_to_export, Mapping):fields = self.fields_to_export.values()else:+ assert self.fields_to_exportfields = self.fields_to_exportrow = list(self._build_row(fields))self.csv_writer.writerow(row)class PickleItemExporter(BaseItemExporter):- def __init__(self, file, protocol=4, **kwargs):+ def __init__(self, file: BytesIO, protocol: int = 4, **kwargs: Any):super().__init__(**kwargs)- self.file = file- self.protocol = protocol+ self.file: BytesIO = file+ self.protocol: int = protocol- def export_item(self, item):+ def export_item(self, item: Any) -> None:d = dict(self._get_serialized_fields(item))pickle.dump(d, self.file, self.protocol)@@ -299,20 +311,20 @@ class MarshalItemExporter(BaseItemExporter):opened in binary mode, a :class:`~io.BytesIO` object, etc)"""- def __init__(self, file, **kwargs):+ def __init__(self, file: BytesIO, **kwargs: Any):super().__init__(**kwargs)- self.file = file+ self.file: BytesIO = file- def export_item(self, item):+ def export_item(self, item: Any) -> None:marshal.dump(dict(self._get_serialized_fields(item)), self.file)class PprintItemExporter(BaseItemExporter):- def __init__(self, file, **kwargs):+ def __init__(self, file: BytesIO, **kwargs: Any):super().__init__(**kwargs)- self.file = file+ self.file: BytesIO = file- def export_item(self, item):+ def export_item(self, item: Any) -> None:itemdict = dict(self._get_serialized_fields(item))self.file.write(to_bytes(pprint.pformat(itemdict) + "\n"))@@ -327,16 +339,20 @@ class PythonItemExporter(BaseItemExporter):.. _msgpack: https://pypi.org/project/msgpack/"""- def _configure(self, options, dont_fail=False):+ def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None:super()._configure(options, dont_fail)if not self.encoding:self.encoding = "utf-8"- def serialize_field(self, field, name, value):- serializer = field.get("serializer", self._serialize_value)+ def serialize_field(+ self, field: Union[Mapping[str, Any], Field], name: str, value: Any+ ) -> Any:+ serializer: Callable[[Any], Any] = field.get(+ "serializer", self._serialize_value+ )return serializer(value)- def _serialize_value(self, value):+ def _serialize_value(self, value: Any) -> Any:if isinstance(value, Item):return self.export_item(value)if is_item(value):@@ -347,10 +363,10 @@ class PythonItemExporter(BaseItemExporter):return to_unicode(value, encoding=self.encoding)return value- def _serialize_item(self, item):+ def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]:for key, value in ItemAdapter(item).items():yield key, self._serialize_value(value)- def export_item(self, item):- result = dict(self._get_serialized_fields(item))+ def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]: # type: ignore[override]+ result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item))return resultdiff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex e5e363b52..3b0dd804e 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -11,7 +11,7 @@ import warningsfrom datetime import datetime, timezonefrom pathlib import Path, PureWindowsPathfrom tempfile import NamedTemporaryFile-from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union+from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Type, Unionfrom urllib.parse import unquote, urlparsefrom twisted.internet import defer, threads@@ -21,6 +21,7 @@ from zope.interface import Interface, implementerfrom scrapy import Spider, signalsfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning+from scrapy.exporters import BaseItemExporterfrom scrapy.extensions.postprocessing import PostProcessingManagerfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.conf import feed_complete_default_values_from_settings@@ -324,12 +325,12 @@ class FeedSlot:filter,feed_options,spider,- exporters,+ exporters: Dict[str, Type[BaseItemExporter]],settings,crawler,):self.file = None- self.exporter = None+ self.exporter: Optional[BaseItemExporter] = Noneself.storage = storage# feed paramsself.batch_id = batch_id@@ -341,7 +342,7 @@ class FeedSlot:# exporter paramsself.feed_options = feed_optionsself.spider = spider- self.exporters = exporters+ self.exporters: Dict[str, Type[BaseItemExporter]] = exportersself.settings = settingsself.crawler = crawler# flags@@ -373,7 +374,7 @@ class FeedSlot:def _get_instance(self, objcls, *args, **kwargs):return build_from_crawler(objcls, self.crawler, *args, **kwargs)- def _get_exporter(self, file, format, *args, **kwargs):+ def _get_exporter(self, file, format, *args, **kwargs) -> BaseItemExporter:return self._get_instance(self.exporters[format], file, *args, **kwargs)def finish_exporting(self):