Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

test: make checkpoint and CSV tests portable to Windows (#457)

diegobravtraps-oss80332cf3a7b3
diff --git a/tests/spiders/test_checkpoint.py b/tests/spiders/test_checkpoint.py
index fc35f43..83acf26 100644
--- a/tests/spiders/test_checkpoint.py
+++ b/tests/spiders/test_checkpoint.py
@@ -56,7 +56,7 @@ class TestCheckpointManagerInit:
"""Test initialization with string path."""
manager = CheckpointManager("/tmp/test_crawl")
- assert str(manager.crawldir) == "/tmp/test_crawl"
+ assert manager.crawldir == Path("/tmp/test_crawl")
assert manager.interval == 300.0
def test_init_with_pathlib_path(self):
@@ -64,7 +64,7 @@ class TestCheckpointManagerInit:
path = Path("/tmp/test_crawl")
manager = CheckpointManager(path)
- assert str(manager.crawldir) == "/tmp/test_crawl"
+ assert manager.crawldir == Path("/tmp/test_crawl")
def test_init_with_custom_interval(self):
"""Test initialization with custom interval."""
@@ -90,8 +90,8 @@ class TestCheckpointManagerInit:
"""Test that checkpoint file path is correctly constructed."""
manager = CheckpointManager("/tmp/test_crawl")
- expected_path = "/tmp/test_crawl/checkpoint.pkl"
- assert str(manager._checkpoint_path) == expected_path
+ expected_path = Path("/tmp/test_crawl/checkpoint.pkl")
+ assert manager._checkpoint_path == expected_path
class TestCheckpointManagerOperations:
diff --git a/tests/spiders/test_result.py b/tests/spiders/test_result.py
index cd8da61..361b84e 100644
--- a/tests/spiders/test_result.py
+++ b/tests/spiders/test_result.py
@@ -87,7 +87,7 @@ class TestItemList:
assert path.exists()
- rows = list(csv.DictReader(path.open(newline="", encoding="utf-8")))
+ rows = list(csv.DictReader(path.read_text(encoding="utf-8").splitlines()))
assert len(rows) == 2
assert rows[0] == {"name": "first", "price": "10"}
assert rows[1]["name"] == "second"
@@ -102,7 +102,7 @@ class TestItemList:
path = Path(tmpdir) / "output.csv"
items.to_csv(path)
- reader = csv.DictReader(path.open(newline="", encoding="utf-8"))
+ reader = csv.DictReader(path.read_text(encoding="utf-8").splitlines())
assert reader.fieldnames == ["name", "price"]
rows = list(reader)
assert rows[0] == {"name": "first", "price": ""}
@@ -116,7 +116,7 @@ class TestItemList:
path = Path(tmpdir) / "output.csv"
items.to_csv(path)
- row = next(csv.DictReader(path.open(newline="", encoding="utf-8")))
+ row = next(csv.DictReader(path.read_text(encoding="utf-8").splitlines()))
assert json.loads(row["tags"]) == ["a", "b"]
assert json.loads(row["meta"]) == {"x": 1}
assert row["empty"] == ""