🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(spiders): add simple export system for the results
Karim shoair72c3c2792a4c
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 26fce1e..8934474 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -5,7 +5,7 @@ from anyio import create_task_group, CapacityLimiterfrom scrapling.core.utils import logfrom scrapling.spiders.request import Request-from scrapling.spiders.result import CrawlStats+from scrapling.spiders.result import CrawlStats, ItemListfrom scrapling.spiders.scheduler import Schedulerfrom scrapling.spiders.session import SessionManagerfrom scrapling.core._types import Dict, TYPE_CHECKING, Any@@ -33,7 +33,7 @@ class CrawlerEngine:self._active_tasks: int = 0self._running: bool = False- self._items: list[dict[str, Any]] = []+ self._items: ItemList = ItemList()def _is_domain_allowed(self, request: Request) -> bool:"""Check if the request's domain is in allowed_domains."""@@ -161,6 +161,6 @@ class CrawlerEngine:return self.stats@property- def items(self) -> list[dict[str, Any]]:+ def items(self) -> ItemList:"""Access scraped items."""return self._itemsdiff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.pyindex d36b0ad..fc10f1b 100644--- a/scrapling/spiders/result.py+++ b/scrapling/spiders/result.py@@ -1,6 +1,42 @@+from pathlib import Pathfrom dataclasses import dataclass, field-from scrapling.core._types import Any, Iterator, Dict, List, Tuple+import orjson++from scrapling.core.utils import log+from scrapling.core._types import Any, Iterator, Dict, List, Tuple, Union+++class ItemList(list):+ """A list of scraped items with export capabilities."""++ def to_json(self, path: Union[str, Path], *, indent: bool = False):+ """Export items to a JSON file.++ :param path: Path to the output file+ :param indent: Pretty-print with 2-space indentation (slightly slower)+ :return: Number of items written+ """+ options = orjson.OPT_SERIALIZE_NUMPY+ if indent:+ options |= orjson.OPT_INDENT_2++ file = Path(path)+ file.parent.mkdir(parents=True, exist_ok=True)+ file.write_bytes(orjson.dumps(list(self), option=options))+ log.info("Saved %d items to %s", len(self), path)++ def to_jsonl(self, path: Union[str, Path]):+ """Export items as JSON Lines (one JSON object per line).++ :param path: Path to the output file+ """+ Path(path).parent.mkdir(parents=True, exist_ok=True)+ with open(path, "wb") as f:+ for item in self:+ f.write(orjson.dumps(item, option=orjson.OPT_SERIALIZE_NUMPY))+ f.write(b"\n")+ log.info("Saved %d items to %s", len(self), path)@dataclass@@ -73,7 +109,7 @@ class CrawlResult:"""Complete result from a spider run."""stats: CrawlStats- items: list[dict[str, Any]]+ items: ItemListdef __len__(self) -> int:return len(self.items)