Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(parser): Add the option to integrate with Scrapy

Karim shoairb2681c295227
diff --git a/docs/integrations/scrapy.md b/docs/integrations/scrapy.md
new file mode 100644
index 0000000..9036626
--- /dev/null
+++ b/docs/integrations/scrapy.md
@@ -0,0 +1,59 @@
+# Scrapy
+
+If you have an existing Scrapy project, you don't need to rewrite it to enjoy Scrapling's parsing API. The Scrapy integration converts Scrapy responses to Scrapling [Response](../fetching/choosing.md#response-object) objects right inside your spider callbacks, so Scrapy keeps handling the crawling while Scrapling handles the parsing.
+
+!!! note "Installation"
+
+ This integration works with the default Scrapling installation (`pip install scrapling`), no extras needed. It only requires Scrapy to be installed, which you already have in a Scrapy project.
+
+## Usage
+
+Put the `scrapling_response` decorator on any spider callback, and the `response` argument it receives becomes a Scrapling `Response`:
+
+```python
+import scrapy
+from scrapling.integrations.scrapy import scrapling_response
+
+
+class QuotesSpider(scrapy.Spider):
+ name = "quotes"
+ start_urls = ["https://quotes.toscrape.com"]
+
+ @scrapling_response
+ def parse(self, response): # `response` is now a Scrapling Response
+ first_quote = response.find_by_text("The world as we have created it", partial=True)
+ for quote in [first_quote, *first_quote.find_similar()]:
+ card = quote.parent
+ yield {
+ "text": quote.get_all_text(strip=True),
+ "author": card.find("small", class_="author").text,
+ "tags": [tag.text for tag in card.find_all("a", class_="tag")],
+ }
+ next_page = response.css("li.next a::attr(href)").get()
+ if next_page:
+ yield scrapy.Request(response.urljoin(next_page), callback=self.parse)
+```
+
+The decorator works on all the callback kinds Scrapy supports: regular functions, generators, coroutines, and async generators. The wrapper keeps the callback's kind, name, and docstring, so Scrapy's callback introspection and contracts keep working.
+
+You can also pass [Selector](../parsing/main_classes.md#selector) configuration to the decorator, and it will be forwarded to the generated `Response`:
+
+```python
+ @scrapling_response(adaptive=True, keep_comments=True)
+ def parse_product(self, response):
+ ...
+```
+
+If you have a Scrapy response at hand outside a callback (middlewares, pipelines, and so on), use the converter directly:
+
+```python
+from scrapling.integrations.scrapy import convert_response
+
+scrapling_response = convert_response(scrapy_response, keep_comments=False, keep_cdata=False)
+```
+
+## Notes
+
+- Yield `scrapy.Request(response.urljoin(href))` for the next pages as in the example above. Scrapling's `Response.follow()` method builds requests for [Scrapling's spider system](../spiders/getting-started.md), which Scrapy doesn't understand.
+- The response's `meta` dictionary is shallow-copied, so objects stored by other middlewares stay reachable. For example, with `scrapy-playwright`, the page is still at `response.meta["playwright_page"]`.
+- Cookies are parsed from the raw `Set-Cookie` headers into the response's `cookies` dictionary.
diff --git a/scrapling/integrations/__init__.py b/scrapling/integrations/__init__.py
new file mode 100644
index 0000000..2666955
--- /dev/null
+++ b/scrapling/integrations/__init__.py
@@ -0,0 +1,7 @@
+"""Integrations with third-party frameworks.
+
+Each integration lives in its own module and is imported explicitly, so its
+framework never becomes a required dependency of Scrapling. Example::
+
+ from scrapling.integrations.scrapy import scrapling_response
+"""
diff --git a/scrapling/integrations/scrapy.py b/scrapling/integrations/scrapy.py
new file mode 100644
index 0000000..ab37a96
--- /dev/null
+++ b/scrapling/integrations/scrapy.py
@@ -0,0 +1,134 @@
+"""Scrapy integration.
+
+Decorate Scrapy spider callbacks with `scrapling_response` to receive a Scrapling `Response`
+object instead of the Scrapy response, so you get Scrapling's full parsing API inside existing
+Scrapy projects without changing how the spider crawls.
+"""
+
+from functools import partial, wraps
+from inspect import isasyncgenfunction, iscoroutinefunction, isgeneratorfunction
+
+from scrapling.core._types import Any, Callable, Dict, Optional, Tuple
+from scrapling.engines.toolbelt.custom import Response, StatusText
+
+try:
+ from scrapy.http import Response as ScrapyResponse
+except (ImportError, ModuleNotFoundError) as e:
+ raise ModuleNotFoundError(
+ "This integration requires Scrapy installed, please install it first with `pip install scrapy`"
+ ) from e
+
+__all__ = ["scrapling_response", "convert_response"]
+
+
+def convert_response(response: ScrapyResponse, **selector_config: Any) -> Response:
+ """Convert a Scrapy response to a Scrapling `Response` object.
+
+ Can be used directly anywhere you have a Scrapy response at hand (middlewares, pipelines, ...).
+
+ :param response: The Scrapy response to convert.
+ :param selector_config: Configuration options passed to the `Response` constructor, like
+ `huge_tree`, `keep_comments`, `keep_cdata`, `adaptive`, `storage`, `storage_args`, and `adaptive_domain`.
+ :return: A Scrapling `Response` object ready for parsing.
+ """
+ request = response.request
+ cookies: Dict[str, str] = {}
+ # `to_unicode_dict` below joins duplicate headers with commas, which corrupts multiple
+ # `Set-Cookie` headers, so cookies are parsed from the raw header lines instead.
+ for line in response.headers.getlist(b"Set-Cookie"):
+ pair = line.split(b";", 1)[0]
+ if b"=" in pair:
+ name, _, value = pair.decode("latin-1").partition("=")
+ cookies[name.strip()] = value.strip()
+
+ return Response(
+ url=response.url,
+ content=response.body,
+ status=response.status,
+ reason=StatusText.get(response.status),
+ cookies=cookies,
+ headers=dict(response.headers.to_unicode_dict()),
+ request_headers=dict(request.headers.to_unicode_dict()) if request else {},
+ encoding=getattr(response, "encoding", "utf-8"), # Binary responses don't have an encoding
+ method=request.method if request else "GET",
+ meta=dict(response.meta) if request else {}, # `meta` raises AttributeError without a request
+ **selector_config,
+ )
+
+
+def scrapling_response(func: Optional[Callable] = None, **selector_config: Any) -> Callable:
+ """Decorator that converts the Scrapy response passed to a spider callback into a Scrapling `Response`.
+
+ Works bare or parameterized, on any callback kind Scrapy supports (regular, generator,
+ coroutine, and async generator functions)::
+
+ class MySpider(scrapy.Spider):
+ @scrapling_response
+ def parse(self, response): # `response` is a Scrapling Response
+ product = response.find_by_text("Vans Old Skool", partial=True)
+ for similar in product.find_similar():
+ yield {"name": similar.get_all_text(strip=True)}
+
+ @scrapling_response(adaptive=True)
+ def parse_product(self, response):
+ ...
+
+ :param func: The decorated callback. Left empty in the parameterized form.
+ :param selector_config: Configuration options passed to the `Response` constructor, like
+ `huge_tree`, `keep_comments`, `keep_cdata`, `adaptive`, `storage`, `storage_args`, and `adaptive_domain`.
+ :return: The wrapped callback. The wrapper keeps the callback's kind, name, and docstring,
+ so Scrapy's callback introspection keeps working.
+ """
+ if func is None:
+ return partial(scrapling_response, **selector_config)
+
+ def _convert_arguments(args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> Tuple[Tuple[Any, ...], Dict[str, Any]]:
+ converted = list(args)
+ for index, argument in enumerate(converted):
+ if isinstance(argument, ScrapyResponse):
+ converted[index] = convert_response(argument, **selector_config)
+ return tuple(converted), kwargs
+
+ for key, value in kwargs.items():
+ if isinstance(value, ScrapyResponse):
+ kwargs[key] = convert_response(value, **selector_config)
+ return tuple(converted), kwargs
+
+ raise TypeError(f"No Scrapy response found in the arguments of '{getattr(func, '__name__', func)}'")
+
+ # Each callback kind gets a wrapper of the same kind because Scrapy inspects the callback
+ # function itself, not just what it returns.
+ if isasyncgenfunction(func):
+
+ @wraps(func)
+ async def async_gen_wrapper(*args: Any, **kwargs: Any) -> Any:
+ args, kwargs = _convert_arguments(args, kwargs)
+ async for result in func(*args, **kwargs):
+ yield result
+
+ return async_gen_wrapper
+
+ elif iscoroutinefunction(func):
+
+ @wraps(func)
+ async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+ args, kwargs = _convert_arguments(args, kwargs)
+ return await func(*args, **kwargs)
+
+ return async_wrapper
+
+ elif isgeneratorfunction(func):
+
+ @wraps(func)
+ def gen_wrapper(*args: Any, **kwargs: Any) -> Any:
+ args, kwargs = _convert_arguments(args, kwargs)
+ yield from func(*args, **kwargs)
+
+ return gen_wrapper
+
+ @wraps(func)
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
+ args, kwargs = _convert_arguments(args, kwargs)
+ return func(*args, **kwargs)
+
+ return wrapper
diff --git a/tests/integrations/__init__.py b/tests/integrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/integrations/test_scrapy.py b/tests/integrations/test_scrapy.py
new file mode 100644
index 0000000..cd038b2
--- /dev/null
+++ b/tests/integrations/test_scrapy.py
@@ -0,0 +1,182 @@
+import inspect
+
+import pytest
+import scrapy
+from scrapy.http import HtmlResponse
+from scrapy.http import Request as ScrapyRequest
+from scrapy.http import Response as ScrapyBaseResponse
+
+from scrapling.engines.toolbelt.custom import Response
+from scrapling.integrations.scrapy import convert_response, scrapling_response
+
+PAGE = b"""<html>
+ <head><title>Test page</title></head>
+ <body>
+ <!-- hidden comment -->
+ <h1>Hello</h1>
+ <a href="/next">Next</a>
+ </body>
+</html>"""
+
+
+def make_response(**kwargs):
+ """Build a Scrapy HtmlResponse with an attached request, no network involved"""
+ if "request" not in kwargs:
+ kwargs["request"] = ScrapyRequest(
+ "http://example.com/page", headers={"User-Agent": "pytest"}, meta={"depth": 1}
+ )
+ defaults = {
+ "url": "http://example.com/page",
+ "body": PAGE,
+ "headers": {"Content-Type": "text/html; charset=utf-8"},
+ "status": 200,
+ }
+ defaults.update(kwargs)
+ return HtmlResponse(**defaults)
+
+
+class TestConvertResponse:
+ """Test converting Scrapy responses to Scrapling responses directly"""
+
+ def test_field_mapping(self):
+ scrapy_response = make_response()
+ response = convert_response(scrapy_response)
+
+ assert isinstance(response, Response)
+ assert response.url == "http://example.com/page"
+ assert response.body == PAGE
+ assert response.status == 200
+ assert response.reason == "OK"
+ assert response.encoding == "utf-8"
+ assert response.headers["Content-Type"] == "text/html; charset=utf-8"
+ assert response.request_headers["User-Agent"] == "pytest"
+ assert response.meta == {"depth": 1}
+ assert response.meta is not scrapy_response.meta
+ assert response.css("h1::text").get() == "Hello"
+ assert response.urljoin("/next") == "http://example.com/next"
+
+ def test_cookies_parsed_from_raw_set_cookie_headers(self):
+ # The `Expires` date contains a comma, which corrupts comma-joined header dicts
+ response = convert_response(
+ make_response(
+ headers={
+ "Set-Cookie": [
+ "sid=abc123; Path=/; Expires=Wed, 09 Jun 2027 10:18:14 GMT",
+ "lang=en; Path=/",
+ ]
+ }
+ )
+ )
+
+ assert response.cookies == {"sid": "abc123", "lang": "en"}
+
+ def test_response_without_request(self):
+ response = convert_response(make_response(request=None))
+
+ assert response.meta == {}
+ assert response.request_headers == {}
+ assert response.css("h1::text").get() == "Hello"
+
+ def test_binary_response_encoding_fallback(self):
+ # The base Scrapy Response class has no `encoding` attribute
+ binary = ScrapyBaseResponse("http://example.com/file.bin", body=b"\x00\x01\x02", status=200)
+ response = convert_response(binary)
+
+ assert response.encoding == "utf-8"
+ assert response.body == b"\x00\x01\x02"
+
+ def test_unknown_status_code_reason(self):
+ response = convert_response(make_response(status=599))
+
+ assert response.status == 599
+ assert response.reason == "Unknown Status Code"
+
+ def test_selector_config_is_forwarded(self):
+ default = convert_response(make_response())
+ with_comments = convert_response(make_response(), keep_comments=True)
+
+ assert not default.xpath("//comment()")
+ assert with_comments.xpath("//comment()")
+
+
+class TestScraplingResponseDecorator:
+ """Test the decorator over all the callback kinds Scrapy supports"""
+
+ def test_generator_callback_on_spider_method(self):
+ class TestSpider(scrapy.Spider):
+ name = "test_spider"
+
+ @scrapling_response
+ def parse(self, response):
+ """Parse the page title"""
+ yield {"title": response.css("title::text").get(), "is_scrapling": isinstance(response, Response)}
+
+ spider = TestSpider()
+ assert inspect.isgeneratorfunction(spider.parse)
+ assert spider.parse.__name__ == "parse"
+ assert spider.parse.__doc__ == "Parse the page title"
+
+ items = list(spider.parse(make_response()))
+ assert items == [{"title": "Test page", "is_scrapling": True}]
+
+ def test_regular_callback(self):
+ @scrapling_response
+ def parse(response):
+ return response.css("h1::text").get()
+
+ assert not inspect.isgeneratorfunction(parse)
+ assert parse(make_response()) == "Hello"
+
+ @pytest.mark.asyncio
+ async def test_coroutine_callback(self):
+ @scrapling_response
+ async def parse(response):
+ return response.css("h1::text").get()
+
+ assert inspect.iscoroutinefunction(parse)
+ assert await parse(make_response()) == "Hello"
+
+ @pytest.mark.asyncio
+ async def test_async_generator_callback(self):
+ @scrapling_response
+ async def parse(response):
+ yield response.css("h1::text").get()
+
+ assert inspect.isasyncgenfunction(parse)
+ assert [item async for item in parse(make_response())] == ["Hello"]
+
+ def test_parameterized_form(self):
+ @scrapling_response(keep_comments=True)
+ def parse(response):
+ return response.xpath("//comment()")
+
+ assert parse(make_response())
+
+ def test_cb_kwargs_passthrough(self):
+ @scrapling_response
+ def parse(response, category=None):
+ return category, response.status
+
+ assert parse(make_response(), category="books") == ("books", 200)
+
+ def test_response_passed_as_keyword(self):
+ @scrapling_response
+ def parse(response):
+ return type(response)
+
+ assert parse(response=make_response()) is Response
+
+ def test_positional_response_wins_over_keyword(self):
+ @scrapling_response
+ def parse(response, other=None):
+ return type(response), type(other)
+
+ assert parse(make_response(), other=make_response()) == (Response, HtmlResponse)
+
+ def test_no_response_in_arguments_raises(self):
+ @scrapling_response
+ def parse(response):
+ return response # pragma: no cover
+
+ with pytest.raises(TypeError, match="No Scrapy response found"):
+ parse("not a response")
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 7c9c3ff..acd9655 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -6,3 +6,4 @@ pytest-httpbin==2.1.0
pytest-asyncio
httpbin~=0.10.0
pytest-xdist
+scrapy
diff --git a/zensical.toml b/zensical.toml
index 70362a5..13550cd 100644
--- a/zensical.toml
+++ b/zensical.toml
@@ -44,7 +44,8 @@ nav = [
{"Extract commands" = "cli/extract-commands.md"}
]},
{Integrations = [
- {"AI MCP server" = "ai/mcp-server.md"}
+ {"AI MCP server" = "ai/mcp-server.md"},
+ {Scrapy = "integrations/scrapy.md"}
]}
]},
{Tutorials = [