Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Speed up exporting scalar values and taking the feed export lock. (#8461)

Andrey Rakhmatullin60c218c2cf06
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 5d4ceb93f..61f0f61c7 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -29,6 +29,8 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
+_SCALAR_TYPES: frozenset[type] = frozenset({bool, bytes, float, int, str, type(None)})
+
__all__ = [
"BaseItemExporter",
"CsvItemExporter",
@@ -81,6 +83,8 @@ class BaseItemExporter(ABC):
return serializer(value)
def _serialize_value(self, value: Any) -> Any:
+ if type(value) in _SCALAR_TYPES:
+ return value
if is_item(value):
adapter = ItemAdapter(value)
field_names = (
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index d72b6e0cd..6f515dbe1 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -962,7 +962,11 @@ class FeedExporter:
)
async def item_scraped(self, item: Any, spider: Spider) -> None:
- await maybe_deferred_to_future(self._export_lock.acquire())
+ if not self._export_lock.locked:
+ # Acquiring a free lock fires the returned Deferred right away
+ self._export_lock.acquire()
+ else:
+ await maybe_deferred_to_future(self._export_lock.acquire())
try:
await self._export_item(item, spider)
finally: