🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(parser): Add the option to integrate with Scrapy
Karim shoairb2681c295227
diff --git a/docs/integrations/scrapy.md b/docs/integrations/scrapy.mdnew file mode 100644index 0000000..9036626--- /dev/null+++ b/docs/integrations/scrapy.md@@ -0,0 +1,59 @@+# Scrapy++If you have an existing Scrapy project, you don't need to rewrite it to enjoy Scrapling's parsing API. The Scrapy integration converts Scrapy responses to Scrapling [Response](../fetching/choosing.md#response-object) objects right inside your spider callbacks, so Scrapy keeps handling the crawling while Scrapling handles the parsing.++!!! note "Installation"++ This integration works with the default Scrapling installation (`pip install scrapling`), no extras needed. It only requires Scrapy to be installed, which you already have in a Scrapy project.++## Usage++Put the `scrapling_response` decorator on any spider callback, and the `response` argument it receives becomes a Scrapling `Response`:++```python+import scrapy+from scrapling.integrations.scrapy import scrapling_response+++class QuotesSpider(scrapy.Spider):+ name = "quotes"+ start_urls = ["https://quotes.toscrape.com"]++ @scrapling_response+ def parse(self, response): # `response` is now a Scrapling Response+ first_quote = response.find_by_text("The world as we have created it", partial=True)+ for quote in [first_quote, *first_quote.find_similar()]:+ card = quote.parent+ yield {+ "text": quote.get_all_text(strip=True),+ "author": card.find("small", class_="author").text,+ "tags": [tag.text for tag in card.find_all("a", class_="tag")],+ }+ next_page = response.css("li.next a::attr(href)").get()+ if next_page:+ yield scrapy.Request(response.urljoin(next_page), callback=self.parse)+```++The decorator works on all the callback kinds Scrapy supports: regular functions, generators, coroutines, and async generators. The wrapper keeps the callback's kind, name, and docstring, so Scrapy's callback introspection and contracts keep working.++You can also pass [Selector](../parsing/main_classes.md#selector) configuration to the decorator, and it will be forwarded to the generated `Response`:++```python+ @scrapling_response(adaptive=True, keep_comments=True)+ def parse_product(self, response):+ ...+```++If you have a Scrapy response at hand outside a callback (middlewares, pipelines, and so on), use the converter directly:++```python+from scrapling.integrations.scrapy import convert_response++scrapling_response = convert_response(scrapy_response, keep_comments=False, keep_cdata=False)+```++## Notes++- Yield `scrapy.Request(response.urljoin(href))` for the next pages as in the example above. Scrapling's `Response.follow()` method builds requests for [Scrapling's spider system](../spiders/getting-started.md), which Scrapy doesn't understand.+- The response's `meta` dictionary is shallow-copied, so objects stored by other middlewares stay reachable. For example, with `scrapy-playwright`, the page is still at `response.meta["playwright_page"]`.+- Cookies are parsed from the raw `Set-Cookie` headers into the response's `cookies` dictionary.diff --git a/scrapling/integrations/__init__.py b/scrapling/integrations/__init__.pynew file mode 100644index 0000000..2666955--- /dev/null+++ b/scrapling/integrations/__init__.py@@ -0,0 +1,7 @@+"""Integrations with third-party frameworks.++Each integration lives in its own module and is imported explicitly, so its+framework never becomes a required dependency of Scrapling. Example::++ from scrapling.integrations.scrapy import scrapling_response+"""diff --git a/scrapling/integrations/scrapy.py b/scrapling/integrations/scrapy.pynew file mode 100644index 0000000..ab37a96--- /dev/null+++ b/scrapling/integrations/scrapy.py@@ -0,0 +1,134 @@+"""Scrapy integration.++Decorate Scrapy spider callbacks with `scrapling_response` to receive a Scrapling `Response`+object instead of the Scrapy response, so you get Scrapling's full parsing API inside existing+Scrapy projects without changing how the spider crawls.+"""++from functools import partial, wraps+from inspect import isasyncgenfunction, iscoroutinefunction, isgeneratorfunction++from scrapling.core._types import Any, Callable, Dict, Optional, Tuple+from scrapling.engines.toolbelt.custom import Response, StatusText++try:+ from scrapy.http import Response as ScrapyResponse+except (ImportError, ModuleNotFoundError) as e:+ raise ModuleNotFoundError(+ "This integration requires Scrapy installed, please install it first with `pip install scrapy`"+ ) from e++__all__ = ["scrapling_response", "convert_response"]+++def convert_response(response: ScrapyResponse, **selector_config: Any) -> Response:+ """Convert a Scrapy response to a Scrapling `Response` object.++ Can be used directly anywhere you have a Scrapy response at hand (middlewares, pipelines, ...).++ :param response: The Scrapy response to convert.+ :param selector_config: Configuration options passed to the `Response` constructor, like+ `huge_tree`, `keep_comments`, `keep_cdata`, `adaptive`, `storage`, `storage_args`, and `adaptive_domain`.+ :return: A Scrapling `Response` object ready for parsing.+ """+ request = response.request+ cookies: Dict[str, str] = {}+ # `to_unicode_dict` below joins duplicate headers with commas, which corrupts multiple+ # `Set-Cookie` headers, so cookies are parsed from the raw header lines instead.+ for line in response.headers.getlist(b"Set-Cookie"):+ pair = line.split(b";", 1)[0]+ if b"=" in pair:+ name, _, value = pair.decode("latin-1").partition("=")+ cookies[name.strip()] = value.strip()++ return Response(+ url=response.url,+ content=response.body,+ status=response.status,+ reason=StatusText.get(response.status),+ cookies=cookies,+ headers=dict(response.headers.to_unicode_dict()),+ request_headers=dict(request.headers.to_unicode_dict()) if request else {},+ encoding=getattr(response, "encoding", "utf-8"), # Binary responses don't have an encoding+ method=request.method if request else "GET",+ meta=dict(response.meta) if request else {}, # `meta` raises AttributeError without a request+ **selector_config,+ )+++def scrapling_response(func: Optional[Callable] = None, **selector_config: Any) -> Callable:+ """Decorator that converts the Scrapy response passed to a spider callback into a Scrapling `Response`.++ Works bare or parameterized, on any callback kind Scrapy supports (regular, generator,+ coroutine, and async generator functions)::++ class MySpider(scrapy.Spider):+ @scrapling_response+ def parse(self, response): # `response` is a Scrapling Response+ product = response.find_by_text("Vans Old Skool", partial=True)+ for similar in product.find_similar():+ yield {"name": similar.get_all_text(strip=True)}++ @scrapling_response(adaptive=True)+ def parse_product(self, response):+ ...++ :param func: The decorated callback. Left empty in the parameterized form.+ :param selector_config: Configuration options passed to the `Response` constructor, like+ `huge_tree`, `keep_comments`, `keep_cdata`, `adaptive`, `storage`, `storage_args`, and `adaptive_domain`.+ :return: The wrapped callback. The wrapper keeps the callback's kind, name, and docstring,+ so Scrapy's callback introspection keeps working.+ """+ if func is None:+ return partial(scrapling_response, **selector_config)++ def _convert_arguments(args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> Tuple[Tuple[Any, ...], Dict[str, Any]]:+ converted = list(args)+ for index, argument in enumerate(converted):+ if isinstance(argument, ScrapyResponse):+ converted[index] = convert_response(argument, **selector_config)+ return tuple(converted), kwargs++ for key, value in kwargs.items():+ if isinstance(value, ScrapyResponse):+ kwargs[key] = convert_response(value, **selector_config)+ return tuple(converted), kwargs++ raise TypeError(f"No Scrapy response found in the arguments of '{getattr(func, '__name__', func)}'")++ # Each callback kind gets a wrapper of the same kind because Scrapy inspects the callback+ # function itself, not just what it returns.+ if isasyncgenfunction(func):++ @wraps(func)+ async def async_gen_wrapper(*args: Any, **kwargs: Any) -> Any:+ args, kwargs = _convert_arguments(args, kwargs)+ async for result in func(*args, **kwargs):+ yield result++ return async_gen_wrapper++ elif iscoroutinefunction(func):++ @wraps(func)+ async def async_wrapper(*args: Any, **kwargs: Any) -> Any:+ args, kwargs = _convert_arguments(args, kwargs)+ return await func(*args, **kwargs)++ return async_wrapper++ elif isgeneratorfunction(func):++ @wraps(func)+ def gen_wrapper(*args: Any, **kwargs: Any) -> Any:+ args, kwargs = _convert_arguments(args, kwargs)+ yield from func(*args, **kwargs)++ return gen_wrapper++ @wraps(func)+ def wrapper(*args: Any, **kwargs: Any) -> Any:+ args, kwargs = _convert_arguments(args, kwargs)+ return func(*args, **kwargs)++ return wrapperdiff --git a/tests/integrations/__init__.py b/tests/integrations/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/tests/integrations/test_scrapy.py b/tests/integrations/test_scrapy.pynew file mode 100644index 0000000..cd038b2--- /dev/null+++ b/tests/integrations/test_scrapy.py@@ -0,0 +1,182 @@+import inspect++import pytest+import scrapy+from scrapy.http import HtmlResponse+from scrapy.http import Request as ScrapyRequest+from scrapy.http import Response as ScrapyBaseResponse++from scrapling.engines.toolbelt.custom import Response+from scrapling.integrations.scrapy import convert_response, scrapling_response++PAGE = b"""<html>+ <head><title>Test page</title></head>+ <body>+ <!-- hidden comment -->+ <h1>Hello</h1>+ <a href="/next">Next</a>+ </body>+</html>"""+++def make_response(**kwargs):+ """Build a Scrapy HtmlResponse with an attached request, no network involved"""+ if "request" not in kwargs:+ kwargs["request"] = ScrapyRequest(+ "http://example.com/page", headers={"User-Agent": "pytest"}, meta={"depth": 1}+ )+ defaults = {+ "url": "http://example.com/page",+ "body": PAGE,+ "headers": {"Content-Type": "text/html; charset=utf-8"},+ "status": 200,+ }+ defaults.update(kwargs)+ return HtmlResponse(**defaults)+++class TestConvertResponse:+ """Test converting Scrapy responses to Scrapling responses directly"""++ def test_field_mapping(self):+ scrapy_response = make_response()+ response = convert_response(scrapy_response)++ assert isinstance(response, Response)+ assert response.url == "http://example.com/page"+ assert response.body == PAGE+ assert response.status == 200+ assert response.reason == "OK"+ assert response.encoding == "utf-8"+ assert response.headers["Content-Type"] == "text/html; charset=utf-8"+ assert response.request_headers["User-Agent"] == "pytest"+ assert response.meta == {"depth": 1}+ assert response.meta is not scrapy_response.meta+ assert response.css("h1::text").get() == "Hello"+ assert response.urljoin("/next") == "http://example.com/next"++ def test_cookies_parsed_from_raw_set_cookie_headers(self):+ # The `Expires` date contains a comma, which corrupts comma-joined header dicts+ response = convert_response(+ make_response(+ headers={+ "Set-Cookie": [+ "sid=abc123; Path=/; Expires=Wed, 09 Jun 2027 10:18:14 GMT",+ "lang=en; Path=/",+ ]+ }+ )+ )++ assert response.cookies == {"sid": "abc123", "lang": "en"}++ def test_response_without_request(self):+ response = convert_response(make_response(request=None))++ assert response.meta == {}+ assert response.request_headers == {}+ assert response.css("h1::text").get() == "Hello"++ def test_binary_response_encoding_fallback(self):+ # The base Scrapy Response class has no `encoding` attribute+ binary = ScrapyBaseResponse("http://example.com/file.bin", body=b"\x00\x01\x02", status=200)+ response = convert_response(binary)++ assert response.encoding == "utf-8"+ assert response.body == b"\x00\x01\x02"++ def test_unknown_status_code_reason(self):+ response = convert_response(make_response(status=599))++ assert response.status == 599+ assert response.reason == "Unknown Status Code"++ def test_selector_config_is_forwarded(self):+ default = convert_response(make_response())+ with_comments = convert_response(make_response(), keep_comments=True)++ assert not default.xpath("//comment()")+ assert with_comments.xpath("//comment()")+++class TestScraplingResponseDecorator:+ """Test the decorator over all the callback kinds Scrapy supports"""++ def test_generator_callback_on_spider_method(self):+ class TestSpider(scrapy.Spider):+ name = "test_spider"++ @scrapling_response+ def parse(self, response):+ """Parse the page title"""+ yield {"title": response.css("title::text").get(), "is_scrapling": isinstance(response, Response)}++ spider = TestSpider()+ assert inspect.isgeneratorfunction(spider.parse)+ assert spider.parse.__name__ == "parse"+ assert spider.parse.__doc__ == "Parse the page title"++ items = list(spider.parse(make_response()))+ assert items == [{"title": "Test page", "is_scrapling": True}]++ def test_regular_callback(self):+ @scrapling_response+ def parse(response):+ return response.css("h1::text").get()++ assert not inspect.isgeneratorfunction(parse)+ assert parse(make_response()) == "Hello"++ @pytest.mark.asyncio+ async def test_coroutine_callback(self):+ @scrapling_response+ async def parse(response):+ return response.css("h1::text").get()++ assert inspect.iscoroutinefunction(parse)+ assert await parse(make_response()) == "Hello"++ @pytest.mark.asyncio+ async def test_async_generator_callback(self):+ @scrapling_response+ async def parse(response):+ yield response.css("h1::text").get()++ assert inspect.isasyncgenfunction(parse)+ assert [item async for item in parse(make_response())] == ["Hello"]++ def test_parameterized_form(self):+ @scrapling_response(keep_comments=True)+ def parse(response):+ return response.xpath("//comment()")++ assert parse(make_response())++ def test_cb_kwargs_passthrough(self):+ @scrapling_response+ def parse(response, category=None):+ return category, response.status++ assert parse(make_response(), category="books") == ("books", 200)++ def test_response_passed_as_keyword(self):+ @scrapling_response+ def parse(response):+ return type(response)++ assert parse(response=make_response()) is Response++ def test_positional_response_wins_over_keyword(self):+ @scrapling_response+ def parse(response, other=None):+ return type(response), type(other)++ assert parse(make_response(), other=make_response()) == (Response, HtmlResponse)++ def test_no_response_in_arguments_raises(self):+ @scrapling_response+ def parse(response):+ return response # pragma: no cover++ with pytest.raises(TypeError, match="No Scrapy response found"):+ parse("not a response")diff --git a/tests/requirements.txt b/tests/requirements.txtindex 7c9c3ff..acd9655 100644--- a/tests/requirements.txt+++ b/tests/requirements.txt@@ -6,3 +6,4 @@ pytest-httpbin==2.1.0pytest-asynciohttpbin~=0.10.0pytest-xdist+scrapydiff --git a/zensical.toml b/zensical.tomlindex 70362a5..13550cd 100644--- a/zensical.toml+++ b/zensical.toml@@ -44,7 +44,8 @@ nav = [{"Extract commands" = "cli/extract-commands.md"}]},{Integrations = [- {"AI MCP server" = "ai/mcp-server.md"}+ {"AI MCP server" = "ai/mcp-server.md"},+ {Scrapy = "integrations/scrapy.md"}]}]},{Tutorials = [