Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import logging
from pathlib import Path
from typing import IO, TYPE_CHECKING, Any
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.feedexport import BlockingFeedStorage, FileFeedStorage
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
from tests.utils.feedexport import crawl_items, path_to_url, printf_escape
if TYPE_CHECKING:
from scrapy.crawler import Crawler
from tests.mockserver.http import MockServer
class DelayedFileStorage(BlockingFeedStorage):
"""Feed storage that, like the S3 or GCS ones, can only detect a conflict
when the feed is delivered, at the end of the crawl or of a batch."""
supported_modes = frozenset({"append", "create", "overwrite"})
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
self.path = Path(uri.split("://", 1)[1])
self.mode = (feed_options or {}).get("mode")
def _store_in_thread(self, file: IO[bytes]) -> None:
file.seek(0)
try:
if self.mode == "create" and self.path.exists():
raise FileExistsError(str(self.path))
self.path.write_bytes(file.read())
finally:
file.close()
class CreateOnlyFileStorage(FileFeedStorage):
supported_modes = frozenset({"create"})
class LegacyFileStorage(FileFeedStorage):
"""Feed storage that predates the mode feed option."""
supported_modes = None # type: ignore[assignment]
class TestFeedMode:
"""End-to-end tests of the mode feed option and the FEED_MODE setting."""
@staticmethod
def _path(tmp_path: Path, content: bytes | None = None) -> Path:
path = tmp_path / "items.jl"
if content is not None:
path.write_bytes(content)
return path
@staticmethod
async def _crawl(
mockserver: MockServer,
path: Path,
mode: str | None = None,
settings: dict[str, Any] | None = None,
item_count: int = 1,
scheme: str | None = None,
) -> Crawler:
feed_options: dict[str, Any] = {"format": "jl"}
if mode is not None:
feed_options["mode"] = mode
if scheme is not None:
# as_posix() keeps the URI valid on Windows, where paths have a
# drive and backslashes.
uri = f"{scheme}://{path.as_posix()}"
elif "%(batch_id)d" in str(path):
# A batch URI template must keep its %(batch_id)d placeholder, so it
# is used as a path, which is neither quoted nor printf-escaped.
uri = str(path)
else:
uri = printf_escape(path_to_url(path))
return await crawl_items(
mockserver,
[{"foo": "bar"}] * item_count,
{"FEEDS": {uri: feed_options}, **(settings or {})},
)
@coroutine_test
async def test_create(self, mockserver: MockServer, tmp_path: Path) -> None:
path = self._path(tmp_path)
await self._crawl(mockserver, path, "create")
assert path.read_bytes() == b'{"foo": "bar"}\n'
@coroutine_test
async def test_create_existing(
self, mockserver: MockServer, tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
path = self._path(tmp_path, b"old content")
with caplog.at_level(logging.ERROR):
crawler = await self._crawl(mockserver, path, "create", item_count=3)
assert path.read_bytes() == b"old content"
# Reported once, not once per item.
assert caplog.text.count("because it already exists") == 1
assert crawler.stats
stats = crawler.stats.get_stats()
assert stats.get("feedexport/conflicts/FileFeedStorage") == 1
assert "feedexport/success_count/FileFeedStorage" not in stats
@coroutine_test
async def test_create_existing_multiple_feeds(
self, mockserver: MockServer, tmp_path: Path
) -> None:
"""A conflicting target only affects its own feed: the crawl runs and
the other feeds are written."""
existing = self._path(tmp_path, b"old content")
missing = tmp_path / "missing.jl"
crawler = await crawl_items(
mockserver,
[{"foo": "bar"}],
{
"FEEDS": {
printf_escape(path_to_url(existing)): {"format": "jl"},
printf_escape(path_to_url(missing)): {"format": "jl"},
},
"FEED_MODE": "create",
},
)
assert existing.read_bytes() == b"old content"
assert missing.read_bytes() == b'{"foo": "bar"}\n'
assert crawler.stats
stats = crawler.stats.get_stats()
assert stats.get("feedexport/conflicts/FileFeedStorage") == 1
assert stats.get("feedexport/success_count/FileFeedStorage") == 1
assert stats.get("finish_reason") == "finished"
@coroutine_test
async def test_create_existing_all_feeds(
self, mockserver: MockServer, tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""A crawl that cannot write any feed still runs, like any other crawl
whose feeds cannot be written."""
existing1 = self._path(tmp_path, b"old content")
existing2 = tmp_path / "items2.jl"
existing2.write_bytes(b"old content")
with caplog.at_level(logging.ERROR):
crawler = await crawl_items(
mockserver,
[{"foo": "bar"}],
{
"FEEDS": {
printf_escape(path_to_url(existing1)): {"format": "jl"},
printf_escape(path_to_url(existing2)): {"format": "jl"},
},
"FEED_MODE": "create",
},
)
assert existing1.read_bytes() == b"old content"
assert existing2.read_bytes() == b"old content"
assert caplog.text.count("because it already exists") == 2
assert crawler.stats
stats = crawler.stats.get_stats()
assert stats.get("feedexport/conflicts/FileFeedStorage") == 2
assert stats.get("finish_reason") == "finished"
assert "feedexport/success_count/FileFeedStorage" not in stats
@coroutine_test
async def test_overwrite_existing(
self, mockserver: MockServer, tmp_path: Path
) -> None:
path = self._path(tmp_path, b"old content")
await self._crawl(mockserver, path, "overwrite")
assert path.read_bytes() == b'{"foo": "bar"}\n'
@coroutine_test
async def test_append_existing(
self, mockserver: MockServer, tmp_path: Path
) -> None:
path = self._path(tmp_path, b"old content\n")
await self._crawl(mockserver, path, "append")
assert path.read_bytes() == b'old content\n{"foo": "bar"}\n'
@coroutine_test
async def test_unset_mode(self, mockserver: MockServer, tmp_path: Path) -> None:
path = self._path(tmp_path)
with pytest.warns(ScrapyDeprecationWarning, match="FEED_MODE"):
await self._crawl(mockserver, path, settings={"FEED_MODE": None})
assert path.read_bytes() == b'{"foo": "bar"}\n'
@coroutine_test
async def test_unset_mode_existing_target(
self, mockserver: MockServer, tmp_path: Path
) -> None:
path = self._path(tmp_path, b"old content\n")
with pytest.warns(ScrapyDeprecationWarning, match="FEED_MODE"):
await self._crawl(mockserver, path, settings={"FEED_MODE": None})
# The legacy behavior is kept.
assert path.read_bytes() == b'old content\n{"foo": "bar"}\n'
@coroutine_test
async def test_explicit_mode_existing_target(
self, mockserver: MockServer, tmp_path: Path, recwarn
) -> None:
path = self._path(tmp_path, b"old content\n")
await self._crawl(mockserver, path, "append")
assert not [
warning
for warning in recwarn
if issubclass(warning.category, ScrapyDeprecationWarning)
and "FEED_MODE" in str(warning.message)
]
@coroutine_test
async def test_create_existing_later_batch(
self, mockserver: MockServer, tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""When the target of a later batch exists, that batch is skipped, and
the following ones are still written."""
(tmp_path / "items-2.jl").write_bytes(b"old content")
with caplog.at_level(logging.ERROR):
crawler = await self._crawl(
mockserver,
tmp_path / "items-%(batch_id)d.jl",
"create",
{"FEED_EXPORT_BATCH_ITEM_COUNT": 1},
item_count=3,
)
assert (tmp_path / "items-1.jl").read_bytes() == b'{"foo": "bar"}\n'
assert (tmp_path / "items-2.jl").read_bytes() == b"old content"
# The skipped items are counted, so the following batch covers the same
# items that it would have covered otherwise.
assert (tmp_path / "items-3.jl").read_bytes() == b'{"foo": "bar"}\n'
assert not (tmp_path / "items-4.jl").exists()
# A single error, instead of one per item after the conflicting one.
assert caplog.text.count("because it already exists") == 1
assert crawler.stats
stats = crawler.stats.get_stats()
assert stats.get("feedexport/conflicts/FileFeedStorage") == 1
assert stats.get("feedexport/success_count/FileFeedStorage") == 2
assert stats.get("finish_reason") == "finished"
@coroutine_test
async def test_create_existing_later_batch_delayed(
self, mockserver: MockServer, tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""When the feed is only delivered at the end of each batch, the
conflict is detected then, and only that batch is lost."""
(tmp_path / "items-2.jl").write_bytes(b"old content")
with caplog.at_level(logging.ERROR):
crawler = await self._crawl(
mockserver,
tmp_path / "items-%(batch_id)d.jl",
"create",
{
"FEED_EXPORT_BATCH_ITEM_COUNT": 1,
"FEED_STORAGES": {"delayed": DelayedFileStorage},
},
item_count=3,
scheme="delayed",
)
assert (tmp_path / "items-1.jl").read_bytes() == b'{"foo": "bar"}\n'
assert (tmp_path / "items-2.jl").read_bytes() == b"old content"
assert (tmp_path / "items-3.jl").read_bytes() == b'{"foo": "bar"}\n'
assert caplog.text.count("because it already exists") == 1
assert crawler.stats
stats = crawler.stats.get_stats()
assert stats.get("feedexport/conflicts/DelayedFileStorage") == 1
assert stats.get("feedexport/failed_count/DelayedFileStorage") == 1
assert stats.get("feedexport/success_count/DelayedFileStorage") == 2
@coroutine_test
async def test_feed_mode_setting(
self, mockserver: MockServer, tmp_path: Path
) -> None:
path = self._path(tmp_path, b"old content")
await self._crawl(mockserver, path, settings={"FEED_MODE": "overwrite"})
assert path.read_bytes() == b'{"foo": "bar"}\n'
@coroutine_test
async def test_mode_overrides_feed_mode_setting(
self, mockserver: MockServer, tmp_path: Path
) -> None:
path = self._path(tmp_path, b"old content\n")
await self._crawl(mockserver, path, "append", {"FEED_MODE": "overwrite"})
assert path.read_bytes() == b'old content\n{"foo": "bar"}\n'
class TestFeedModeInit:
def test_invalid_mode(self) -> None:
"""An invalid mode prevents the crawl from running."""
settings = {
"FEEDS": {"file:///tmp/items.json": {"format": "json", "mode": "x"}}
}
with pytest.raises(ValueError, match="Invalid feed mode: 'x'"):
get_crawler(settings_dict=settings)
def test_unsupported_mode(self) -> None:
settings = {
"FEEDS": {"file:///tmp/items.json": {"format": "json"}},
"FEED_STORAGES": {"file": CreateOnlyFileStorage},
"FEED_MODE": "append",
}
with pytest.raises(
ValueError,
match="CreateOnlyFileStorage does not support the 'append' feed mode",
):
get_crawler(settings_dict=settings)
def test_undeclared_mode_support(self, caplog: pytest.LogCaptureFixture) -> None:
settings = {
"FEEDS": {"file:///tmp/items.json": {"format": "json"}},
"FEED_STORAGES": {"file": LegacyFileStorage},
"FEED_MODE": "create",
}
with caplog.at_level(logging.WARNING):
get_crawler(settings_dict=settings)
assert "does not declare which feed modes it supports" in caplog.text
def test_undeclared_mode_support_unset_mode(
self, caplog: pytest.LogCaptureFixture
) -> None:
settings = {
"FEEDS": {"file:///tmp/items.json": {"format": "json"}},
"FEED_STORAGES": {"file": LegacyFileStorage},
"FEED_MODE": None,
}
with (
caplog.at_level(logging.WARNING),
pytest.warns(ScrapyDeprecationWarning, match="FEED_MODE"),
):
get_crawler(settings_dict=settings)
assert "does not declare which feed modes it supports" not in caplog.text