Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate and disable escape_ajax().

Andrey Rakhmatullin4842bcbf1da4
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index c6a55732d..e7a8962a1 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -9,6 +9,7 @@ from w3lib import html
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Response
+from scrapy.utils.url import escape_ajax
if TYPE_CHECKING:
# typing.Self requires Python 3.11
@@ -64,8 +65,7 @@ class AjaxCrawlMiddleware:
if not self._has_ajax_crawlable_variant(response):
return response
- # scrapy already handles #! links properly
- ajax_crawl_request = request.replace(url=request.url + "#!")
+ ajax_crawl_request = request.replace(url=escape_ajax(request.url + "#!"))
logger.debug(
"Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
{"ajax_crawl_request": ajax_crawl_request, "request": request},
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 4eee5ffbb..e24f6874d 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -27,7 +27,6 @@ from scrapy.http.headers import Headers
from scrapy.utils.curl import curl_to_request_kwargs
from scrapy.utils.python import to_bytes
from scrapy.utils.trackref import object_ref
-from scrapy.utils.url import escape_ajax
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Mapping
@@ -170,8 +169,7 @@ class Request(object_ref):
if not isinstance(url, str):
raise TypeError(f"Request url must be str, got {type(url).__name__}")
- s = safe_url_string(url, self.encoding)
- self._url = escape_ajax(s)
+ self._url = safe_url_string(url, self.encoding)
if (
"://" not in self._url
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index db2749d79..1348cc992 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -10,6 +10,7 @@ import warnings
from importlib import import_module
from typing import TYPE_CHECKING, Union
from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
+from warnings import warn
from w3lib.url import __all__ as _public_w3lib_objects
from w3lib.url import add_or_replace_parameter as _add_or_replace_parameter
@@ -83,6 +84,11 @@ def escape_ajax(url: str) -> str:
>>> escape_ajax("www.example.com/ajax.html")
'www.example.com/ajax.html'
"""
+ warn(
+ "escape_ajax() is deprecated and will be removed in a future Scrapy version.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
defrag, frag = urldefrag(url)
if not frag.startswith("!"):
return url
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 9915aaca4..a8ab8240f 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -187,18 +187,6 @@ class RequestTest(unittest.TestCase):
assert isinstance(r4.body, bytes)
self.assertEqual(r4.body, b"Price: \xa3100")
- def test_ajax_url(self):
- # ascii url
- r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
- self.assertEqual(
- r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue"
- )
- # unicode url
- r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
- self.assertEqual(
- r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue"
- )
-
def test_copy(self):
"""Test Request copy"""