🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat: add a template for Shopify websites
Karim shoair4022dac08677
diff --git a/scrapling/spiders/__init__.py b/scrapling/spiders/__init__.pyindex b455076..5eda5fd 100644--- a/scrapling/spiders/__init__.py+++ b/scrapling/spiders/__init__.py@@ -5,7 +5,7 @@ from .engine import CrawlerEnginefrom .session import SessionManagerfrom .spider import Spider, SessionConfigurationErrorfrom .links import LinkExtractor-from .templates import CrawlSpider, SitemapSpider, CrawlRule+from .templates import CrawlSpider, SitemapSpider, CrawlRule, ShopifySpiderfrom scrapling.engines.toolbelt.custom import Response__all__ = [@@ -21,4 +21,5 @@ __all__ = ["CrawlSpider","CrawlRule","SitemapSpider",+ "ShopifySpider",]diff --git a/scrapling/spiders/templates/__init__.py b/scrapling/spiders/templates/__init__.pyindex fa758c7..b556701 100644--- a/scrapling/spiders/templates/__init__.py+++ b/scrapling/spiders/templates/__init__.py@@ -1,8 +1,10 @@from .crawler import CrawlSpider, CrawlRulefrom .sitemap import SitemapSpider+from .shopify import ShopifySpider__all__ = ["CrawlSpider","CrawlRule","SitemapSpider",+ "ShopifySpider",]diff --git a/scrapling/spiders/templates/shopify.py b/scrapling/spiders/templates/shopify.pynew file mode 100644index 0000000..ed6b361--- /dev/null+++ b/scrapling/spiders/templates/shopify.py@@ -0,0 +1,112 @@+"""Spider template for extracting products from Shopify-powered websites."""++from urllib.parse import urlparse++from w3lib.html import remove_tags++from scrapling.spiders.request import Request+from scrapling.spiders.spider import Spider+from scrapling.core._types import (+ TYPE_CHECKING,+ Any,+ AsyncGenerator,+ Dict,+ Generator,+ Set,+ Union,+)++if TYPE_CHECKING:+ from scrapling.engines.toolbelt.custom import Response++__all__ = ["ShopifySpider"]+++class ShopifySpider(Spider):+ """A spider that extracts all products from any Shopify-powered website through its JSON API.++ Set `target_website` to the store's domain (or set `start_urls`/`allowed_domains` instead), and the+ spider walks the store's `/collections.json` pages, then each collection's `products.json` pages,+ yielding one item per product variant without touching the website's HTML.+ """++ name = "shopify"+ target_website = ""+ collections_url = "https://{website}/collections.json?page={page}&limit=250"+ products_url = "https://{website}/collections/{handle}/products.json?page={page}&limit=250"+ product_url = "https://{website}/collections/{handle}/products/{product_handle}"++ def __init__(self, *args: Any, **kwargs: Any):+ super().__init__(*args, **kwargs)+ source = self.target_website or next(iter(self.start_urls or ()), "") or next(iter(self.allowed_domains), "")+ if not source:+ raise ValueError(f"{self.__class__.__name__} must set `target_website`, `start_urls`, or `allowed_domains`")+ self.target_website = urlparse(source if "://" in source else f"https://{source}").netloc+ self.collected_ids: Set[int] = set()++ async def start_requests(self) -> AsyncGenerator[Request, None]:+ yield Request(+ self.collections_url.format(website=self.target_website, page=1),+ callback=self.parse,+ meta={"page": 1},+ )++ async def parse(self, response: "Response") -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]:+ collections = response.json()["collections"]+ if collections:+ for collection in collections:+ if collection["products_count"]:+ yield Request(+ self.products_url.format(website=self.target_website, handle=collection["handle"], page=1),+ callback=self.parse_collection,+ meta={"handle": collection["handle"], "page": 1},+ )++ next_page = response.meta.get("page", 1) + 1+ yield Request(+ self.collections_url.format(website=self.target_website, page=next_page),+ callback=self.parse,+ meta={"page": next_page},+ )++ def _process_product(self, product: Dict, collection_handle: str) -> Generator[Dict[str, Any], None, None]:+ for variant in product["variants"]:+ if variant["id"] in self.collected_ids:+ continue++ self.collected_ids.add(variant["id"])+ old_price = variant.get("compare_at_price")+ yield {+ "name": product["title"] + (f" - {variant['title']}" if variant["title"] != "Default Title" else ""),+ "price": variant["price"],+ "category": collection_handle.replace("-", " ").title().strip(),+ "brand": product["vendor"],+ "identifier": variant["id"],+ "sku": variant.get("sku") or "",+ "stock": None if variant.get("available") else 0,+ "image_url": product["images"][0]["src"] if product["images"] else "",+ "url": self.product_url.format(+ website=self.target_website, handle=collection_handle, product_handle=product["handle"]+ ),+ "description": remove_tags(product.get("body_html") or ""),+ "old_price": old_price if old_price and float(old_price) else "",+ "barcode": variant.get("barcode") or "",+ }++ async def parse_collection(+ self, response: "Response"+ ) -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]:+ collection_handle, current_page = response.meta["handle"], response.meta.get("page", 1)+ products = response.json()["products"]+ if products:+ for product in products:+ for item in self._process_product(product, collection_handle):+ yield item++ yield Request(+ self.products_url.format(website=self.target_website, handle=collection_handle, page=current_page + 1),+ callback=self.parse_collection,+ meta={"handle": collection_handle, "page": current_page + 1},+ )+ else:+ self.logger.debug(f"Extracted all products from collection {collection_handle}")diff --git a/tests/spiders/test_shopify.py b/tests/spiders/test_shopify.pynew file mode 100644index 0000000..bfb69e9--- /dev/null+++ b/tests/spiders/test_shopify.py@@ -0,0 +1,212 @@+"""Tests for `ShopifySpider`."""++import orjson+import pytest++from scrapling.engines.toolbelt.custom import Response+from scrapling.spiders.request import Request+from scrapling.spiders.templates import ShopifySpider+from scrapling.core._types import Any, Dict+++COLLECTIONS_PAGE = {+ "collections": [+ {"handle": "lipsticks", "title": "Lipsticks", "products_count": 2},+ {"handle": "empty-collection", "title": "Empty", "products_count": 0},+ ]+}++PRODUCTS_PAGE = {+ "products": [+ {+ "id": 1,+ "title": "Matte Lipstick",+ "handle": "matte-lipstick",+ "vendor": "Acme",+ "body_html": "<p>A <b>matte</b> lipstick</p>",+ "images": [{"src": "https://cdn.shopify.com/lipstick.jpg"}],+ "variants": [+ {+ "id": 11,+ "title": "Red",+ "sku": "LIP-RED",+ "price": "9.00",+ "available": True,+ "compare_at_price": "12.00",+ },+ {+ "id": 12,+ "title": "Nude",+ "sku": None,+ "price": "9.00",+ "available": False,+ "compare_at_price": "0.00",+ },+ ],+ },+ {+ "id": 2,+ "title": "Brow Pencil",+ "handle": "brow-pencil",+ "vendor": "Acme",+ "body_html": None,+ "images": [],+ "variants": [+ {"id": 21, "title": "Default Title", "sku": "BROW-1", "price": "7.50", "available": True},+ ],+ },+ ]+}+++def _make_response(url: str, payload: Dict[str, Any], meta: Dict[str, Any] | None = None) -> Response:+ resp = Response(+ url=url,+ content=orjson.dumps(payload),+ status=200,+ reason="OK",+ cookies={},+ headers={},+ request_headers={},+ )+ resp.request = Request(url, meta=meta or {})+ resp.meta.update(meta or {})+ return resp+++class ExampleStoreSpider(ShopifySpider):+ target_website = "example.com"+++class TestDomainResolution:+ def test_from_target_website(self):+ assert ExampleStoreSpider().target_website == "example.com"++ def test_target_website_normalized_from_url(self):+ class S(ShopifySpider):+ target_website = "https://example.com/collections/all"++ assert S().target_website == "example.com"++ def test_from_start_urls(self):+ class S(ShopifySpider):+ start_urls = ["https://www.example.com/collections/mens"]++ assert S().target_website == "www.example.com"++ def test_from_allowed_domains(self):+ class S(ShopifySpider):+ allowed_domains = {"example.com"}++ assert S().target_website == "example.com"++ def test_no_source_raises(self):+ with pytest.raises(ValueError, match="must set"):+ ShopifySpider()+++class TestParsing:+ @pytest.mark.asyncio+ async def test_start_requests(self):+ spider = ExampleStoreSpider()+ requests = [r async for r in spider.start_requests()]+ assert len(requests) == 1+ assert requests[0].url == "https://example.com/collections.json?page=1&limit=250"+ assert requests[0].meta == {"page": 1}++ @pytest.mark.asyncio+ async def test_parse_dispatches_collections_and_next_page(self):+ spider = ExampleStoreSpider()+ response = _make_response(+ "https://example.com/collections.json?page=1&limit=250", COLLECTIONS_PAGE, {"page": 1}+ )+ results = [r async for r in spider.parse(response)]++ assert all(isinstance(r, Request) for r in results)+ assert len(results) == 2+ assert results[0].url == "https://example.com/collections/lipsticks/products.json?page=1&limit=250"+ assert results[0].meta == {"handle": "lipsticks", "page": 1}+ assert results[1].url == "https://example.com/collections.json?page=2&limit=250"+ assert results[1].meta == {"page": 2}++ @pytest.mark.asyncio+ async def test_parse_stops_on_empty_page(self):+ spider = ExampleStoreSpider()+ response = _make_response(+ "https://example.com/collections.json?page=3&limit=250", {"collections": []}, {"page": 3}+ )+ assert [r async for r in spider.parse(response)] == []++ @pytest.mark.asyncio+ async def test_parse_collection_yields_items_and_next_page(self):+ spider = ExampleStoreSpider()+ response = _make_response(+ "https://example.com/collections/lipsticks/products.json?page=1&limit=250",+ PRODUCTS_PAGE,+ {"handle": "lipsticks", "page": 1},+ )+ results = [r async for r in spider.parse_collection(response)]+ items = [r for r in results if isinstance(r, dict)]+ requests = [r for r in results if isinstance(r, Request)]++ assert len(items) == 3+ assert len(requests) == 1+ assert requests[0].url == "https://example.com/collections/lipsticks/products.json?page=2&limit=250"+ assert requests[0].meta == {"handle": "lipsticks", "page": 2}++ @pytest.mark.asyncio+ async def test_parse_collection_stops_on_empty_page(self):+ spider = ExampleStoreSpider()+ response = _make_response(+ "https://example.com/collections/lipsticks/products.json?page=2&limit=250",+ {"products": []},+ {"handle": "lipsticks", "page": 2},+ )+ assert [r async for r in spider.parse_collection(response)] == []+++class TestItemProcessing:+ def test_variant_expansion_and_fields(self):+ spider = ExampleStoreSpider()+ items = list(spider._process_product(PRODUCTS_PAGE["products"][0], "lipsticks"))++ assert len(items) == 2+ assert items[0] == {+ "name": "Matte Lipstick - Red",+ "price": "9.00",+ "category": "Lipsticks",+ "brand": "Acme",+ "identifier": 11,+ "sku": "LIP-RED",+ "stock": None,+ "image_url": "https://cdn.shopify.com/lipstick.jpg",+ "url": "https://example.com/collections/lipsticks/products/matte-lipstick",+ "description": "A matte lipstick",+ "old_price": "12.00",+ "barcode": "",+ }+ assert items[1]["sku"] == ""+ assert items[1]["stock"] == 0+ assert items[1]["old_price"] == ""++ def test_default_title_variant_keeps_product_name(self):+ spider = ExampleStoreSpider()+ items = list(spider._process_product(PRODUCTS_PAGE["products"][1], "brow-pencils"))++ assert len(items) == 1+ assert items[0]["name"] == "Brow Pencil"+ assert items[0]["category"] == "Brow Pencils"+ assert items[0]["image_url"] == ""+ assert items[0]["description"] == ""++ def test_variants_deduplicated_across_collections(self):+ spider = ExampleStoreSpider()+ product = PRODUCTS_PAGE["products"][0]++ assert len(list(spider._process_product(product, "lipsticks"))) == 2+ assert list(spider._process_product(product, "bestsellers")) == []++ def test_dedup_state_is_per_instance(self):+ product = PRODUCTS_PAGE["products"][0]+ assert len(list(ExampleStoreSpider()._process_product(product, "lipsticks"))) == 2+ assert len(list(ExampleStoreSpider()._process_product(product, "lipsticks"))) == 2