Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat: add a template for Shopify websites

Karim shoair4022dac08677
diff --git a/scrapling/spiders/__init__.py b/scrapling/spiders/__init__.py
index b455076..5eda5fd 100644
--- a/scrapling/spiders/__init__.py
+++ b/scrapling/spiders/__init__.py
@@ -5,7 +5,7 @@ from .engine import CrawlerEngine
from .session import SessionManager
from .spider import Spider, SessionConfigurationError
from .links import LinkExtractor
-from .templates import CrawlSpider, SitemapSpider, CrawlRule
+from .templates import CrawlSpider, SitemapSpider, CrawlRule, ShopifySpider
from scrapling.engines.toolbelt.custom import Response
__all__ = [
@@ -21,4 +21,5 @@ __all__ = [
"CrawlSpider",
"CrawlRule",
"SitemapSpider",
+ "ShopifySpider",
]
diff --git a/scrapling/spiders/templates/__init__.py b/scrapling/spiders/templates/__init__.py
index fa758c7..b556701 100644
--- a/scrapling/spiders/templates/__init__.py
+++ b/scrapling/spiders/templates/__init__.py
@@ -1,8 +1,10 @@
from .crawler import CrawlSpider, CrawlRule
from .sitemap import SitemapSpider
+from .shopify import ShopifySpider
__all__ = [
"CrawlSpider",
"CrawlRule",
"SitemapSpider",
+ "ShopifySpider",
]
diff --git a/scrapling/spiders/templates/shopify.py b/scrapling/spiders/templates/shopify.py
new file mode 100644
index 0000000..ed6b361
--- /dev/null
+++ b/scrapling/spiders/templates/shopify.py
@@ -0,0 +1,112 @@
+"""Spider template for extracting products from Shopify-powered websites."""
+
+from urllib.parse import urlparse
+
+from w3lib.html import remove_tags
+
+from scrapling.spiders.request import Request
+from scrapling.spiders.spider import Spider
+from scrapling.core._types import (
+ TYPE_CHECKING,
+ Any,
+ AsyncGenerator,
+ Dict,
+ Generator,
+ Set,
+ Union,
+)
+
+if TYPE_CHECKING:
+ from scrapling.engines.toolbelt.custom import Response
+
+__all__ = ["ShopifySpider"]
+
+
+class ShopifySpider(Spider):
+ """A spider that extracts all products from any Shopify-powered website through its JSON API.
+
+ Set `target_website` to the store's domain (or set `start_urls`/`allowed_domains` instead), and the
+ spider walks the store's `/collections.json` pages, then each collection's `products.json` pages,
+ yielding one item per product variant without touching the website's HTML.
+ """
+
+ name = "shopify"
+ target_website = ""
+ collections_url = "https://{website}/collections.json?page={page}&limit=250"
+ products_url = "https://{website}/collections/{handle}/products.json?page={page}&limit=250"
+ product_url = "https://{website}/collections/{handle}/products/{product_handle}"
+
+ def __init__(self, *args: Any, **kwargs: Any):
+ super().__init__(*args, **kwargs)
+ source = self.target_website or next(iter(self.start_urls or ()), "") or next(iter(self.allowed_domains), "")
+ if not source:
+ raise ValueError(f"{self.__class__.__name__} must set `target_website`, `start_urls`, or `allowed_domains`")
+ self.target_website = urlparse(source if "://" in source else f"https://{source}").netloc
+ self.collected_ids: Set[int] = set()
+
+ async def start_requests(self) -> AsyncGenerator[Request, None]:
+ yield Request(
+ self.collections_url.format(website=self.target_website, page=1),
+ callback=self.parse,
+ meta={"page": 1},
+ )
+
+ async def parse(self, response: "Response") -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]:
+ collections = response.json()["collections"]
+ if collections:
+ for collection in collections:
+ if collection["products_count"]:
+ yield Request(
+ self.products_url.format(website=self.target_website, handle=collection["handle"], page=1),
+ callback=self.parse_collection,
+ meta={"handle": collection["handle"], "page": 1},
+ )
+
+ next_page = response.meta.get("page", 1) + 1
+ yield Request(
+ self.collections_url.format(website=self.target_website, page=next_page),
+ callback=self.parse,
+ meta={"page": next_page},
+ )
+
+ def _process_product(self, product: Dict, collection_handle: str) -> Generator[Dict[str, Any], None, None]:
+ for variant in product["variants"]:
+ if variant["id"] in self.collected_ids:
+ continue
+
+ self.collected_ids.add(variant["id"])
+ old_price = variant.get("compare_at_price")
+ yield {
+ "name": product["title"] + (f" - {variant['title']}" if variant["title"] != "Default Title" else ""),
+ "price": variant["price"],
+ "category": collection_handle.replace("-", " ").title().strip(),
+ "brand": product["vendor"],
+ "identifier": variant["id"],
+ "sku": variant.get("sku") or "",
+ "stock": None if variant.get("available") else 0,
+ "image_url": product["images"][0]["src"] if product["images"] else "",
+ "url": self.product_url.format(
+ website=self.target_website, handle=collection_handle, product_handle=product["handle"]
+ ),
+ "description": remove_tags(product.get("body_html") or ""),
+ "old_price": old_price if old_price and float(old_price) else "",
+ "barcode": variant.get("barcode") or "",
+ }
+
+ async def parse_collection(
+ self, response: "Response"
+ ) -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]:
+ collection_handle, current_page = response.meta["handle"], response.meta.get("page", 1)
+ products = response.json()["products"]
+ if products:
+ for product in products:
+ for item in self._process_product(product, collection_handle):
+ yield item
+
+ yield Request(
+ self.products_url.format(website=self.target_website, handle=collection_handle, page=current_page + 1),
+ callback=self.parse_collection,
+ meta={"handle": collection_handle, "page": current_page + 1},
+ )
+ else:
+ self.logger.debug(f"Extracted all products from collection {collection_handle}")
diff --git a/tests/spiders/test_shopify.py b/tests/spiders/test_shopify.py
new file mode 100644
index 0000000..bfb69e9
--- /dev/null
+++ b/tests/spiders/test_shopify.py
@@ -0,0 +1,212 @@
+"""Tests for `ShopifySpider`."""
+
+import orjson
+import pytest
+
+from scrapling.engines.toolbelt.custom import Response
+from scrapling.spiders.request import Request
+from scrapling.spiders.templates import ShopifySpider
+from scrapling.core._types import Any, Dict
+
+
+COLLECTIONS_PAGE = {
+ "collections": [
+ {"handle": "lipsticks", "title": "Lipsticks", "products_count": 2},
+ {"handle": "empty-collection", "title": "Empty", "products_count": 0},
+ ]
+}
+
+PRODUCTS_PAGE = {
+ "products": [
+ {
+ "id": 1,
+ "title": "Matte Lipstick",
+ "handle": "matte-lipstick",
+ "vendor": "Acme",
+ "body_html": "<p>A <b>matte</b> lipstick</p>",
+ "images": [{"src": "https://cdn.shopify.com/lipstick.jpg"}],
+ "variants": [
+ {
+ "id": 11,
+ "title": "Red",
+ "sku": "LIP-RED",
+ "price": "9.00",
+ "available": True,
+ "compare_at_price": "12.00",
+ },
+ {
+ "id": 12,
+ "title": "Nude",
+ "sku": None,
+ "price": "9.00",
+ "available": False,
+ "compare_at_price": "0.00",
+ },
+ ],
+ },
+ {
+ "id": 2,
+ "title": "Brow Pencil",
+ "handle": "brow-pencil",
+ "vendor": "Acme",
+ "body_html": None,
+ "images": [],
+ "variants": [
+ {"id": 21, "title": "Default Title", "sku": "BROW-1", "price": "7.50", "available": True},
+ ],
+ },
+ ]
+}
+
+
+def _make_response(url: str, payload: Dict[str, Any], meta: Dict[str, Any] | None = None) -> Response:
+ resp = Response(
+ url=url,
+ content=orjson.dumps(payload),
+ status=200,
+ reason="OK",
+ cookies={},
+ headers={},
+ request_headers={},
+ )
+ resp.request = Request(url, meta=meta or {})
+ resp.meta.update(meta or {})
+ return resp
+
+
+class ExampleStoreSpider(ShopifySpider):
+ target_website = "example.com"
+
+
+class TestDomainResolution:
+ def test_from_target_website(self):
+ assert ExampleStoreSpider().target_website == "example.com"
+
+ def test_target_website_normalized_from_url(self):
+ class S(ShopifySpider):
+ target_website = "https://example.com/collections/all"
+
+ assert S().target_website == "example.com"
+
+ def test_from_start_urls(self):
+ class S(ShopifySpider):
+ start_urls = ["https://www.example.com/collections/mens"]
+
+ assert S().target_website == "www.example.com"
+
+ def test_from_allowed_domains(self):
+ class S(ShopifySpider):
+ allowed_domains = {"example.com"}
+
+ assert S().target_website == "example.com"
+
+ def test_no_source_raises(self):
+ with pytest.raises(ValueError, match="must set"):
+ ShopifySpider()
+
+
+class TestParsing:
+ @pytest.mark.asyncio
+ async def test_start_requests(self):
+ spider = ExampleStoreSpider()
+ requests = [r async for r in spider.start_requests()]
+ assert len(requests) == 1
+ assert requests[0].url == "https://example.com/collections.json?page=1&limit=250"
+ assert requests[0].meta == {"page": 1}
+
+ @pytest.mark.asyncio
+ async def test_parse_dispatches_collections_and_next_page(self):
+ spider = ExampleStoreSpider()
+ response = _make_response(
+ "https://example.com/collections.json?page=1&limit=250", COLLECTIONS_PAGE, {"page": 1}
+ )
+ results = [r async for r in spider.parse(response)]
+
+ assert all(isinstance(r, Request) for r in results)
+ assert len(results) == 2
+ assert results[0].url == "https://example.com/collections/lipsticks/products.json?page=1&limit=250"
+ assert results[0].meta == {"handle": "lipsticks", "page": 1}
+ assert results[1].url == "https://example.com/collections.json?page=2&limit=250"
+ assert results[1].meta == {"page": 2}
+
+ @pytest.mark.asyncio
+ async def test_parse_stops_on_empty_page(self):
+ spider = ExampleStoreSpider()
+ response = _make_response(
+ "https://example.com/collections.json?page=3&limit=250", {"collections": []}, {"page": 3}
+ )
+ assert [r async for r in spider.parse(response)] == []
+
+ @pytest.mark.asyncio
+ async def test_parse_collection_yields_items_and_next_page(self):
+ spider = ExampleStoreSpider()
+ response = _make_response(
+ "https://example.com/collections/lipsticks/products.json?page=1&limit=250",
+ PRODUCTS_PAGE,
+ {"handle": "lipsticks", "page": 1},
+ )
+ results = [r async for r in spider.parse_collection(response)]
+ items = [r for r in results if isinstance(r, dict)]
+ requests = [r for r in results if isinstance(r, Request)]
+
+ assert len(items) == 3
+ assert len(requests) == 1
+ assert requests[0].url == "https://example.com/collections/lipsticks/products.json?page=2&limit=250"
+ assert requests[0].meta == {"handle": "lipsticks", "page": 2}
+
+ @pytest.mark.asyncio
+ async def test_parse_collection_stops_on_empty_page(self):
+ spider = ExampleStoreSpider()
+ response = _make_response(
+ "https://example.com/collections/lipsticks/products.json?page=2&limit=250",
+ {"products": []},
+ {"handle": "lipsticks", "page": 2},
+ )
+ assert [r async for r in spider.parse_collection(response)] == []
+
+
+class TestItemProcessing:
+ def test_variant_expansion_and_fields(self):
+ spider = ExampleStoreSpider()
+ items = list(spider._process_product(PRODUCTS_PAGE["products"][0], "lipsticks"))
+
+ assert len(items) == 2
+ assert items[0] == {
+ "name": "Matte Lipstick - Red",
+ "price": "9.00",
+ "category": "Lipsticks",
+ "brand": "Acme",
+ "identifier": 11,
+ "sku": "LIP-RED",
+ "stock": None,
+ "image_url": "https://cdn.shopify.com/lipstick.jpg",
+ "url": "https://example.com/collections/lipsticks/products/matte-lipstick",
+ "description": "A matte lipstick",
+ "old_price": "12.00",
+ "barcode": "",
+ }
+ assert items[1]["sku"] == ""
+ assert items[1]["stock"] == 0
+ assert items[1]["old_price"] == ""
+
+ def test_default_title_variant_keeps_product_name(self):
+ spider = ExampleStoreSpider()
+ items = list(spider._process_product(PRODUCTS_PAGE["products"][1], "brow-pencils"))
+
+ assert len(items) == 1
+ assert items[0]["name"] == "Brow Pencil"
+ assert items[0]["category"] == "Brow Pencils"
+ assert items[0]["image_url"] == ""
+ assert items[0]["description"] == ""
+
+ def test_variants_deduplicated_across_collections(self):
+ spider = ExampleStoreSpider()
+ product = PRODUCTS_PAGE["products"][0]
+
+ assert len(list(spider._process_product(product, "lipsticks"))) == 2
+ assert list(spider._process_product(product, "bestsellers")) == []
+
+ def test_dedup_state_is_per_instance(self):
+ product = PRODUCTS_PAGE["products"][0]
+ assert len(list(ExampleStoreSpider()._process_product(product, "lipsticks"))) == 2
+ assert len(list(ExampleStoreSpider()._process_product(product, "lipsticks"))) == 2