🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: stop response.follow() inheriting the method, params and body (#462)
Yuval Dinodia1ac8e52fb1ea
diff --git a/agent-skill/Scrapling-Skill/references/spiders/requests-responses.md b/agent-skill/Scrapling-Skill/references/spiders/requests-responses.mdindex c93a7e2..bb0ab70 100644--- a/agent-skill/Scrapling-Skill/references/spiders/requests-responses.md+++ b/agent-skill/Scrapling-Skill/references/spiders/requests-responses.md@@ -49,7 +49,7 @@ yield Request(- **Relative URLs** are resolved automatically against the current page URL- **Referer header** is set to the current page URL by default-- **Session kwargs** from the original request are inherited (headers, proxy settings, etc.)+- **Session kwargs** from the original request are inherited (headers, proxy settings, etc.), except `method`, `params`, `data`, `json`, `files`, and `multipart`, so followed links are plain GETs- **Callback, session ID, and priority** are inherited from the original request if not specified```pythondiff --git a/agent-skill/Scrapling-Skill/references/spiders/sessions.md b/agent-skill/Scrapling-Skill/references/spiders/sessions.mdindex 3428b8d..8ab54ea 100644--- a/agent-skill/Scrapling-Skill/references/spiders/sessions.md+++ b/agent-skill/Scrapling-Skill/references/spiders/sessions.md@@ -182,7 +182,7 @@ async def parse(self, response: Response):)```-**Warning:** Session arguments (**kwargs) passed from the original request are inherited by `response.follow()`. New kwargs take precedence over inherited ones.+**Warning:** Session arguments (**kwargs) passed from the original request are inherited by `response.follow()`, except `method`, `params`, `data`, `json`, `files`, and `multipart`, so following a link after a POST issues a plain GET. New kwargs take precedence over inherited ones.```pythonfrom scrapling.spiders import Spider, Responsediff --git a/docs/spiders/requests-responses.md b/docs/spiders/requests-responses.mdindex f49fd53..397d53a 100644--- a/docs/spiders/requests-responses.md+++ b/docs/spiders/requests-responses.md@@ -53,7 +53,7 @@ yield Request(- **Relative URLs** are resolved automatically against the current page URL- **Referer header** is set to the current page URL by default-- **Session kwargs** from the original request are inherited (headers, proxy settings, etc.)+- **Session kwargs** from the original request are inherited (headers, proxy settings, etc.), except `method`, `params`, `data`, `json`, `files`, and `multipart`, so followed links are plain GETs- **Callback, session ID, and priority** are inherited from the original request if not specified```pythondiff --git a/docs/spiders/sessions.md b/docs/spiders/sessions.mdindex d62d72e..caccafd 100644--- a/docs/spiders/sessions.md+++ b/docs/spiders/sessions.md@@ -193,7 +193,7 @@ async def parse(self, response: Response):!!! warning- Session arguments (**kwargs) passed from the original request are inherited by `response.follow()`. New kwargs take precedence over inherited ones.+ Session arguments (**kwargs) passed from the original request are inherited by `response.follow()`, except `method`, `params`, `data`, `json`, `files`, and `multipart`, so following a link after a POST issues a plain GET. New kwargs take precedence over inherited ones.```pythonfrom scrapling.spiders import Spider, Responsediff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 0527844..f6c0c0f 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -25,6 +25,9 @@ if TYPE_CHECKING:from scrapling.spiders import Request+_NON_INHERITABLE_KWARGS = frozenset({"method", "data", "json", "params", "files", "multipart"})++class Response(Selector):"""This class is returned by all engines as a way to unify the response type between different libraries.@@ -119,6 +122,8 @@ class Response(Selector):**IMPORTANT**: The below arguments if left empty, the corresponding value from the previous request will be used. The only exception is `dont_filter`.+ The session kwargs of the previous request are inherited as well, except `method`, `params`, `data`, `json`, `files`, and `multipart`, so a followed link is a plain GET unless you pass them again.+:param url: The URL to follow (can be relative, will be joined with current URL):param sid: The session id to use:param callback: Spider callback method to use@@ -135,7 +140,11 @@ class Response(Selector):raise TypeError("This response has no request set yet.")# Merge original session kwargs with new kwargs (new takes precedence)- session_kwargs = {**self.request._session_kwargs, **kwargs}+ inherited = {k: v for k, v in self.request._session_kwargs.items() if k not in _NON_INHERITABLE_KWARGS}+ for key in ("headers", "extra_headers"):+ if inherited.get(key):+ inherited[key] = {k: v for k, v in inherited[key].items() if k.lower() != "content-length"}+ session_kwargs = {**inherited, **kwargs}if referer_flow:# For requestsdiff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.pyindex e2f3767..df580b5 100644--- a/tests/spiders/test_request.py+++ b/tests/spiders/test_request.py@@ -5,6 +5,7 @@ import pickleimport pytestfrom scrapling.spiders.request import Request+from scrapling.engines.toolbelt.custom import Responsefrom scrapling.core._types import Any, Dict, AsyncGenerator@@ -446,3 +447,95 @@ class TestRequestRestoreCallback:# Should not raise an errorrequest._restore_callback(spider) # type: ignore[arg-type]+++class TestResponseFollow:+ def _response(self, request: Request) -> Response:+ response = Response(+ url=request.url,+ content=b"",+ status=200,+ reason="OK",+ cookies={},+ headers={},+ request_headers={},+ )+ response.request = request+ return response++ def test_follow_does_not_inherit_method_or_payload(self):+ parent = Request(+ "https://example.com/login",+ method="POST",+ data={"username": "admin", "password": "secret"},+ json={"key": "value"},+ params={"page": "1"},+ files={"file": b"content"},+ multipart="mime",+ proxy="http://proxy:8080",+ timeout=30,+ )++ followed = self._response(parent).follow("/page/2/?page=2", referer_flow=False)++ assert followed.url == "https://example.com/page/2/?page=2"+ assert followed._session_kwargs == {"proxy": "http://proxy:8080", "timeout": 30}++ def test_follow_to_another_host_is_a_plain_get(self):+ parent = Request(+ "https://example.com/login",+ method="POST",+ data={"username": "admin", "password": "secret"},+ params={"page": "1"},+ proxy="http://proxy:8080",+ )++ followed = self._response(parent).follow("https://httpbin.org/anything?page=2")++ assert followed.url == "https://httpbin.org/anything?page=2"+ assert followed._session_kwargs == {+ "proxy": "http://proxy:8080",+ "headers": {"referer": "https://example.com/login"},+ "extra_headers": {"referer": "https://example.com/login"},+ "google_search": False,+ }++ @pytest.mark.parametrize(+ "override",+ [+ {"method": "POST", "data": {"q": "books"}},+ {"method": "POST", "json": {"q": "books"}},+ {"params": {"q": "books"}},+ {"method": "POST", "files": {"file": b"content"}},+ {"method": "POST", "multipart": "mime"},+ ],+ )+ def test_follow_keeps_explicit_overrides(self, override):+ parent = Request("https://example.com/login", method="POST", data={"password": "secret"})++ followed = self._response(parent).follow("/search", referer_flow=False, **override)++ assert followed._session_kwargs == override++ def test_follow_drops_inherited_content_length(self):+ parent = Request(+ "https://example.com/login",+ method="POST",+ data={"password": "secret"},+ headers={"Content-Length": "15", "X-Token": "abc"},+ extra_headers={"content-length": "15"},+ )++ followed = self._response(parent).follow("/search", referer_flow=False)++ assert followed._session_kwargs == {"headers": {"X-Token": "abc"}, "extra_headers": {}}+ assert parent._session_kwargs["headers"] == {"Content-Length": "15", "X-Token": "abc"}++ def test_follow_keeps_explicit_content_length(self):+ parent = Request("https://example.com/login", method="POST", headers={"Content-Length": "15"})++ followed = self._response(parent).follow(+ "/search", referer_flow=False, method="POST", data="q=books", headers={"Content-Length": "7"}+ )++ assert followed._session_kwargs == {"method": "POST", "data": "q=books", "headers": {"Content-Length": "7"}}