Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): corrections and removal of dead code

Karim shoair02b11c535e1f
diff --git a/scrapling/spiders/checkpoint.py b/scrapling/spiders/checkpoint.py
index 4c511ae..aa84432 100644
--- a/scrapling/spiders/checkpoint.py
+++ b/scrapling/spiders/checkpoint.py
@@ -32,7 +32,7 @@ class CheckpointManager:
if not isinstance(interval, (int, float)):
raise TypeError("Checkpoints interval must be integer or float.")
else:
- if interval < 0:
+ if interval <= 0:
raise ValueError("Checkpoints interval must be greater than 0.")
async def has_checkpoint(self) -> bool:
diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py
index edfc387..ce728ee 100644
--- a/scrapling/spiders/request.py
+++ b/scrapling/spiders/request.py
@@ -85,8 +85,6 @@ class Request:
body = post_data.getvalue()
elif isinstance(post_data, bytes):
body = post_data
- elif post_data is None:
- body = b""
else:
post_data = self._session_kwargs.get("json", {})
body = orjson.dumps(post_data) if post_data else b""
diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.py
index eeb5522..08a7658 100644
--- a/scrapling/spiders/result.py
+++ b/scrapling/spiders/result.py
@@ -15,7 +15,6 @@ class ItemList(list):
:param path: Path to the output file
:param indent: Pretty-print with 2-space indentation (slightly slower)
- :return: Number of items written
"""
options = orjson.OPT_SERIALIZE_NUMPY
if indent:
diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py
index c994d2b..4f38912 100644
--- a/scrapling/spiders/spider.py
+++ b/scrapling/spiders/spider.py
@@ -79,9 +79,9 @@ class Spider(ABC):
max_blocked_retries: int = 3
# Fingerprint adjustments
- fp_include_kwargs = False
- fp_keep_fragments = False
- fp_include_headers = False
+ fp_include_kwargs: bool = False
+ fp_keep_fragments: bool = False
+ fp_include_headers: bool = False
# Logging settings
logging_level: int = logging.DEBUG