Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: stop response.follow() inheriting the method, params and body (#462)

Yuval Dinodia1ac8e52fb1ea
diff --git a/agent-skill/Scrapling-Skill/references/spiders/requests-responses.md b/agent-skill/Scrapling-Skill/references/spiders/requests-responses.md
index c93a7e2..bb0ab70 100644
--- a/agent-skill/Scrapling-Skill/references/spiders/requests-responses.md
+++ b/agent-skill/Scrapling-Skill/references/spiders/requests-responses.md
@@ -49,7 +49,7 @@ yield Request(
- **Relative URLs** are resolved automatically against the current page URL
- **Referer header** is set to the current page URL by default
-- **Session kwargs** from the original request are inherited (headers, proxy settings, etc.)
+- **Session kwargs** from the original request are inherited (headers, proxy settings, etc.), except `method`, `params`, `data`, `json`, `files`, and `multipart`, so followed links are plain GETs
- **Callback, session ID, and priority** are inherited from the original request if not specified
```python
diff --git a/agent-skill/Scrapling-Skill/references/spiders/sessions.md b/agent-skill/Scrapling-Skill/references/spiders/sessions.md
index 3428b8d..8ab54ea 100644
--- a/agent-skill/Scrapling-Skill/references/spiders/sessions.md
+++ b/agent-skill/Scrapling-Skill/references/spiders/sessions.md
@@ -182,7 +182,7 @@ async def parse(self, response: Response):
)
```
-**Warning:** Session arguments (**kwargs) passed from the original request are inherited by `response.follow()`. New kwargs take precedence over inherited ones.
+**Warning:** Session arguments (**kwargs) passed from the original request are inherited by `response.follow()`, except `method`, `params`, `data`, `json`, `files`, and `multipart`, so following a link after a POST issues a plain GET. New kwargs take precedence over inherited ones.
```python
from scrapling.spiders import Spider, Response
diff --git a/docs/spiders/requests-responses.md b/docs/spiders/requests-responses.md
index f49fd53..397d53a 100644
--- a/docs/spiders/requests-responses.md
+++ b/docs/spiders/requests-responses.md
@@ -53,7 +53,7 @@ yield Request(
- **Relative URLs** are resolved automatically against the current page URL
- **Referer header** is set to the current page URL by default
-- **Session kwargs** from the original request are inherited (headers, proxy settings, etc.)
+- **Session kwargs** from the original request are inherited (headers, proxy settings, etc.), except `method`, `params`, `data`, `json`, `files`, and `multipart`, so followed links are plain GETs
- **Callback, session ID, and priority** are inherited from the original request if not specified
```python
diff --git a/docs/spiders/sessions.md b/docs/spiders/sessions.md
index d62d72e..caccafd 100644
--- a/docs/spiders/sessions.md
+++ b/docs/spiders/sessions.md
@@ -193,7 +193,7 @@ async def parse(self, response: Response):
!!! warning
- Session arguments (**kwargs) passed from the original request are inherited by `response.follow()`. New kwargs take precedence over inherited ones.
+ Session arguments (**kwargs) passed from the original request are inherited by `response.follow()`, except `method`, `params`, `data`, `json`, `files`, and `multipart`, so following a link after a POST issues a plain GET. New kwargs take precedence over inherited ones.
```python
from scrapling.spiders import Spider, Response
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 0527844..f6c0c0f 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -25,6 +25,9 @@ if TYPE_CHECKING:
from scrapling.spiders import Request
+_NON_INHERITABLE_KWARGS = frozenset({"method", "data", "json", "params", "files", "multipart"})
+
+
class Response(Selector):
"""This class is returned by all engines as a way to unify the response type between different libraries.
@@ -119,6 +122,8 @@ class Response(Selector):
**IMPORTANT**: The below arguments if left empty, the corresponding value from the previous request will be used. The only exception is `dont_filter`.
+ The session kwargs of the previous request are inherited as well, except `method`, `params`, `data`, `json`, `files`, and `multipart`, so a followed link is a plain GET unless you pass them again.
+
:param url: The URL to follow (can be relative, will be joined with current URL)
:param sid: The session id to use
:param callback: Spider callback method to use
@@ -135,7 +140,11 @@ class Response(Selector):
raise TypeError("This response has no request set yet.")
# Merge original session kwargs with new kwargs (new takes precedence)
- session_kwargs = {**self.request._session_kwargs, **kwargs}
+ inherited = {k: v for k, v in self.request._session_kwargs.items() if k not in _NON_INHERITABLE_KWARGS}
+ for key in ("headers", "extra_headers"):
+ if inherited.get(key):
+ inherited[key] = {k: v for k, v in inherited[key].items() if k.lower() != "content-length"}
+ session_kwargs = {**inherited, **kwargs}
if referer_flow:
# For requests
diff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.py
index e2f3767..df580b5 100644
--- a/tests/spiders/test_request.py
+++ b/tests/spiders/test_request.py
@@ -5,6 +5,7 @@ import pickle
import pytest
from scrapling.spiders.request import Request
+from scrapling.engines.toolbelt.custom import Response
from scrapling.core._types import Any, Dict, AsyncGenerator
@@ -446,3 +447,95 @@ class TestRequestRestoreCallback:
# Should not raise an error
request._restore_callback(spider) # type: ignore[arg-type]
+
+
+class TestResponseFollow:
+ def _response(self, request: Request) -> Response:
+ response = Response(
+ url=request.url,
+ content=b"",
+ status=200,
+ reason="OK",
+ cookies={},
+ headers={},
+ request_headers={},
+ )
+ response.request = request
+ return response
+
+ def test_follow_does_not_inherit_method_or_payload(self):
+ parent = Request(
+ "https://example.com/login",
+ method="POST",
+ data={"username": "admin", "password": "secret"},
+ json={"key": "value"},
+ params={"page": "1"},
+ files={"file": b"content"},
+ multipart="mime",
+ proxy="http://proxy:8080",
+ timeout=30,
+ )
+
+ followed = self._response(parent).follow("/page/2/?page=2", referer_flow=False)
+
+ assert followed.url == "https://example.com/page/2/?page=2"
+ assert followed._session_kwargs == {"proxy": "http://proxy:8080", "timeout": 30}
+
+ def test_follow_to_another_host_is_a_plain_get(self):
+ parent = Request(
+ "https://example.com/login",
+ method="POST",
+ data={"username": "admin", "password": "secret"},
+ params={"page": "1"},
+ proxy="http://proxy:8080",
+ )
+
+ followed = self._response(parent).follow("https://httpbin.org/anything?page=2")
+
+ assert followed.url == "https://httpbin.org/anything?page=2"
+ assert followed._session_kwargs == {
+ "proxy": "http://proxy:8080",
+ "headers": {"referer": "https://example.com/login"},
+ "extra_headers": {"referer": "https://example.com/login"},
+ "google_search": False,
+ }
+
+ @pytest.mark.parametrize(
+ "override",
+ [
+ {"method": "POST", "data": {"q": "books"}},
+ {"method": "POST", "json": {"q": "books"}},
+ {"params": {"q": "books"}},
+ {"method": "POST", "files": {"file": b"content"}},
+ {"method": "POST", "multipart": "mime"},
+ ],
+ )
+ def test_follow_keeps_explicit_overrides(self, override):
+ parent = Request("https://example.com/login", method="POST", data={"password": "secret"})
+
+ followed = self._response(parent).follow("/search", referer_flow=False, **override)
+
+ assert followed._session_kwargs == override
+
+ def test_follow_drops_inherited_content_length(self):
+ parent = Request(
+ "https://example.com/login",
+ method="POST",
+ data={"password": "secret"},
+ headers={"Content-Length": "15", "X-Token": "abc"},
+ extra_headers={"content-length": "15"},
+ )
+
+ followed = self._response(parent).follow("/search", referer_flow=False)
+
+ assert followed._session_kwargs == {"headers": {"X-Token": "abc"}, "extra_headers": {}}
+ assert parent._session_kwargs["headers"] == {"Content-Length": "15", "X-Token": "abc"}
+
+ def test_follow_keeps_explicit_content_length(self):
+ parent = Request("https://example.com/login", method="POST", headers={"Content-Length": "15"})
+
+ followed = self._response(parent).follow(
+ "/search", referer_flow=False, method="POST", data="q=books", headers={"Content-Length": "7"}
+ )
+
+ assert followed._session_kwargs == {"method": "POST", "data": "q=books", "headers": {"Content-Length": "7"}}