Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): use os.replace for atomic checkpoint/cache writes on Windows

Ahmed Elshahatf0db3d7d1405
diff --git a/scrapling/spiders/cache.py b/scrapling/spiders/cache.py
index 40d39d3..0305aef 100644
--- a/scrapling/spiders/cache.py
+++ b/scrapling/spiders/cache.py
@@ -64,7 +64,7 @@ class ResponseCacheManager:
async with await anyio.open_file(temp_path, "wb") as f:
await f.write(serialized)
- await temp_path.rename(self._cache_path(fingerprint))
+ await temp_path.replace(self._cache_path(fingerprint))
except Exception as e:
if await temp_path.exists():
await temp_path.unlink()
diff --git a/scrapling/spiders/checkpoint.py b/scrapling/spiders/checkpoint.py
index 25de362..95515bf 100644
--- a/scrapling/spiders/checkpoint.py
+++ b/scrapling/spiders/checkpoint.py
@@ -50,7 +50,7 @@ class CheckpointManager:
async with await anyio.open_file(temp_path, "wb") as f:
await f.write(serialized)
- await temp_path.rename(self._checkpoint_path)
+ await temp_path.replace(self._checkpoint_path)
log.info(f"Checkpoint saved: {len(data.requests)} requests, {len(data.seen)} seen URLs")
except Exception as e:
diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.py
index fc9bc59..eb1b04b 100644
--- a/tests/spiders/test_cache.py
+++ b/tests/spiders/test_cache.py
@@ -49,6 +49,27 @@ class TestResponseCacheManager:
assert dict(restored.headers) == dict(original.headers)
assert dict(restored.request_headers) == dict(original.request_headers)
+ @pytest.mark.anyio
+ async def test_put_overwrites_existing_entry(self):
+ """Re-caching the same fingerprint must replace the stored response.
+
+ Regression test for a Windows-only failure: ``Path.rename`` cannot
+ overwrite an existing destination on Windows (raising ``WinError 183``),
+ so the second ``put`` was caught by the error handler, the temp file was
+ removed, and ``get`` kept returning the stale body. ``Path.replace``
+ overwrites atomically on every platform.
+ """
+ with tempfile.TemporaryDirectory() as tmpdir:
+ cache = ResponseCacheManager(tmpdir)
+ fp = b"\x05" * 20
+
+ await cache.put(fp, _make_response(body=b"<html>first</html>"), "GET")
+ await cache.put(fp, _make_response(body=b"<html>second</html>"), "GET")
+
+ restored = await cache.get(fp)
+ assert restored is not None
+ assert restored.body == b"<html>second</html>"
+
@pytest.mark.anyio
async def test_get_cache_miss(self):
with tempfile.TemporaryDirectory() as tmpdir: