🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): corrections and removal of dead code
Karim shoair02b11c535e1f
diff --git a/scrapling/spiders/checkpoint.py b/scrapling/spiders/checkpoint.pyindex 4c511ae..aa84432 100644--- a/scrapling/spiders/checkpoint.py+++ b/scrapling/spiders/checkpoint.py@@ -32,7 +32,7 @@ class CheckpointManager:if not isinstance(interval, (int, float)):raise TypeError("Checkpoints interval must be integer or float.")else:- if interval < 0:+ if interval <= 0:raise ValueError("Checkpoints interval must be greater than 0.")async def has_checkpoint(self) -> bool:diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.pyindex edfc387..ce728ee 100644--- a/scrapling/spiders/request.py+++ b/scrapling/spiders/request.py@@ -85,8 +85,6 @@ class Request:body = post_data.getvalue()elif isinstance(post_data, bytes):body = post_data- elif post_data is None:- body = b""else:post_data = self._session_kwargs.get("json", {})body = orjson.dumps(post_data) if post_data else b""diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.pyindex eeb5522..08a7658 100644--- a/scrapling/spiders/result.py+++ b/scrapling/spiders/result.py@@ -15,7 +15,6 @@ class ItemList(list)::param path: Path to the output file:param indent: Pretty-print with 2-space indentation (slightly slower)- :return: Number of items written"""options = orjson.OPT_SERIALIZE_NUMPYif indent:diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.pyindex c994d2b..4f38912 100644--- a/scrapling/spiders/spider.py+++ b/scrapling/spiders/spider.py@@ -79,9 +79,9 @@ class Spider(ABC):max_blocked_retries: int = 3# Fingerprint adjustments- fp_include_kwargs = False- fp_keep_fragments = False- fp_include_headers = False+ fp_include_kwargs: bool = False+ fp_keep_fragments: bool = False+ fp_include_headers: bool = False# Logging settingslogging_level: int = logging.DEBUG