Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import logging
import tempfile
from logging import getLogger
from typing import TYPE_CHECKING
import pytest
from w3lib.url import file_uri_to_path
from tests.utils.decorators import coroutine_test
from tests.utils.feedexport import export_by_format, unique_path
if TYPE_CHECKING:
from pathlib import Path
from tests.mockserver.http import MockServer
class LogOnStoreFileStorage:
"""
This storage logs inside `store` method.
It can be used to make sure `store` method is invoked.
"""
def __init__(self, uri, feed_options=None):
self.path = file_uri_to_path(uri)
self.logger = getLogger()
def open(self, spider):
return tempfile.NamedTemporaryFile(prefix="feed-")
def store(self, file):
self.logger.info("Storage.store is called")
file.close()
@pytest.mark.parametrize("fmt", ["json", "jsonlines", "xml", "csv"])
@coroutine_test
async def test_export_no_items_not_store_empty(
fmt: str, mockserver: MockServer, tmp_path: Path
) -> None:
settings = {
"FEEDS": {
unique_path(tmp_path): {"format": fmt},
},
"FEED_STORE_EMPTY": False,
}
data = await export_by_format(mockserver, [], settings)
assert data[fmt] is None
@pytest.mark.parametrize(
("fmt", "expected"),
[
pytest.param("json", b"[]", id="json"),
pytest.param("jsonlines", b"", id="jsonlines"),
pytest.param(
"xml",
b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>',
id="xml",
),
pytest.param("csv", b"", id="csv"),
],
)
@coroutine_test
async def test_export_no_items_store_empty(
fmt: str, expected: bytes, mockserver: MockServer, tmp_path: Path
) -> None:
settings = {
"FEEDS": {
unique_path(tmp_path): {"format": fmt},
},
"FEED_STORE_EMPTY": True,
"FEED_EXPORT_INDENT": None,
}
data = await export_by_format(mockserver, [], settings)
assert data[fmt] == expected
@coroutine_test
async def test_export_no_items_multiple_feeds(
caplog: pytest.LogCaptureFixture, mockserver: MockServer, tmp_path: Path
) -> None:
"""Make sure that `storage.store` is not called."""
settings = {
"FEEDS": {
unique_path(tmp_path): {"format": "json"},
unique_path(tmp_path): {"format": "xml"},
unique_path(tmp_path): {"format": "csv"},
},
"FEED_STORAGES": {"file": LogOnStoreFileStorage},
"FEED_STORE_EMPTY": False,
}
with caplog.at_level(logging.INFO):
await export_by_format(mockserver, [], settings)
assert caplog.text.count("Storage.store is called") == 0