Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(spiders): add simple export system for the results

Karim shoair72c3c2792a4c
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 26fce1e..8934474 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -5,7 +5,7 @@ from anyio import create_task_group, CapacityLimiter
from scrapling.core.utils import log
from scrapling.spiders.request import Request
-from scrapling.spiders.result import CrawlStats
+from scrapling.spiders.result import CrawlStats, ItemList
from scrapling.spiders.scheduler import Scheduler
from scrapling.spiders.session import SessionManager
from scrapling.core._types import Dict, TYPE_CHECKING, Any
@@ -33,7 +33,7 @@ class CrawlerEngine:
self._active_tasks: int = 0
self._running: bool = False
- self._items: list[dict[str, Any]] = []
+ self._items: ItemList = ItemList()
def _is_domain_allowed(self, request: Request) -> bool:
"""Check if the request's domain is in allowed_domains."""
@@ -161,6 +161,6 @@ class CrawlerEngine:
return self.stats
@property
- def items(self) -> list[dict[str, Any]]:
+ def items(self) -> ItemList:
"""Access scraped items."""
return self._items
diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.py
index d36b0ad..fc10f1b 100644
--- a/scrapling/spiders/result.py
+++ b/scrapling/spiders/result.py
@@ -1,6 +1,42 @@
+from pathlib import Path
from dataclasses import dataclass, field
-from scrapling.core._types import Any, Iterator, Dict, List, Tuple
+import orjson
+
+from scrapling.core.utils import log
+from scrapling.core._types import Any, Iterator, Dict, List, Tuple, Union
+
+
+class ItemList(list):
+ """A list of scraped items with export capabilities."""
+
+ def to_json(self, path: Union[str, Path], *, indent: bool = False):
+ """Export items to a JSON file.
+
+ :param path: Path to the output file
+ :param indent: Pretty-print with 2-space indentation (slightly slower)
+ :return: Number of items written
+ """
+ options = orjson.OPT_SERIALIZE_NUMPY
+ if indent:
+ options |= orjson.OPT_INDENT_2
+
+ file = Path(path)
+ file.parent.mkdir(parents=True, exist_ok=True)
+ file.write_bytes(orjson.dumps(list(self), option=options))
+ log.info("Saved %d items to %s", len(self), path)
+
+ def to_jsonl(self, path: Union[str, Path]):
+ """Export items as JSON Lines (one JSON object per line).
+
+ :param path: Path to the output file
+ """
+ Path(path).parent.mkdir(parents=True, exist_ok=True)
+ with open(path, "wb") as f:
+ for item in self:
+ f.write(orjson.dumps(item, option=orjson.OPT_SERIALIZE_NUMPY))
+ f.write(b"\n")
+ log.info("Saved %d items to %s", len(self), path)
@dataclass
@@ -73,7 +109,7 @@ class CrawlResult:
"""Complete result from a spider run."""
stats: CrawlStats
- items: list[dict[str, Any]]
+ items: ItemList
def __len__(self) -> int:
return len(self.items)