🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): use os.replace for atomic checkpoint/cache writes on Windows
Ahmed Elshahatf0db3d7d1405
diff --git a/scrapling/spiders/cache.py b/scrapling/spiders/cache.pyindex 40d39d3..0305aef 100644--- a/scrapling/spiders/cache.py+++ b/scrapling/spiders/cache.py@@ -64,7 +64,7 @@ class ResponseCacheManager:async with await anyio.open_file(temp_path, "wb") as f:await f.write(serialized)- await temp_path.rename(self._cache_path(fingerprint))+ await temp_path.replace(self._cache_path(fingerprint))except Exception as e:if await temp_path.exists():await temp_path.unlink()diff --git a/scrapling/spiders/checkpoint.py b/scrapling/spiders/checkpoint.pyindex 25de362..95515bf 100644--- a/scrapling/spiders/checkpoint.py+++ b/scrapling/spiders/checkpoint.py@@ -50,7 +50,7 @@ class CheckpointManager:async with await anyio.open_file(temp_path, "wb") as f:await f.write(serialized)- await temp_path.rename(self._checkpoint_path)+ await temp_path.replace(self._checkpoint_path)log.info(f"Checkpoint saved: {len(data.requests)} requests, {len(data.seen)} seen URLs")except Exception as e:diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.pyindex fc9bc59..eb1b04b 100644--- a/tests/spiders/test_cache.py+++ b/tests/spiders/test_cache.py@@ -49,6 +49,27 @@ class TestResponseCacheManager:assert dict(restored.headers) == dict(original.headers)assert dict(restored.request_headers) == dict(original.request_headers)+ @pytest.mark.anyio+ async def test_put_overwrites_existing_entry(self):+ """Re-caching the same fingerprint must replace the stored response.++ Regression test for a Windows-only failure: ``Path.rename`` cannot+ overwrite an existing destination on Windows (raising ``WinError 183``),+ so the second ``put`` was caught by the error handler, the temp file was+ removed, and ``get`` kept returning the stale body. ``Path.replace``+ overwrites atomically on every platform.+ """+ with tempfile.TemporaryDirectory() as tmpdir:+ cache = ResponseCacheManager(tmpdir)+ fp = b"\x05" * 20++ await cache.put(fp, _make_response(body=b"<html>first</html>"), "GET")+ await cache.put(fp, _make_response(body=b"<html>second</html>"), "GET")++ restored = await cache.get(fp)+ assert restored is not None+ assert restored.body == b"<html>second</html>"+@pytest.mark.anyioasync def test_get_cache_miss(self):with tempfile.TemporaryDirectory() as tmpdir: