Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate AjaxCrawlMiddleware.
Andrey Rakhmatullin393ff96e45ff
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rstindex 0286c3354..248e38b61 100644--- a/docs/topics/broad-crawls.rst+++ b/docs/topics/broad-crawls.rst@@ -182,30 +182,6 @@ To disable redirects use:REDIRECT_ENABLED = False-Enable crawling of "Ajax Crawlable Pages"-=========================================--Some pages (up to 1%, based on empirical data from year 2013) declare-themselves as ajax crawlable. This means they provide plain HTML-version of content that is usually available only via AJAX.-Pages can indicate it in two ways:--1) by using ``#!`` in URL - this is the default way;-2) by using a special meta tag - this way is used on- "main", "index" website pages.--Scrapy handles (1) automatically; to handle (2) enable-:ref:`AjaxCrawlMiddleware <ajaxcrawl-middleware>`:--.. code-block:: python-- AJAXCRAWL_ENABLED = True--When doing broad crawls it's common to crawl a lot of "index" web pages;-AjaxCrawlMiddleware helps to crawl them correctly.-It is turned OFF by default because it has some performance overhead,-and enabling it for focused crawls doesn't make much sense.-.. _broad-crawls-bfo:Crawl in BFO orderdiff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 1ab8f588f..ca597291f 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -1249,8 +1249,7 @@ AJAXCRAWL_ENABLEDDefault: ``False``-Whether the AjaxCrawlMiddleware will be enabled. You may want to-enable it for :ref:`broad crawls <topics-broad-crawls>`.+Whether the AjaxCrawlMiddleware will be enabled.HttpProxyMiddleware settings~~~~~~~~~~~~~~~~~~~~~~~~~~~~diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.pyindex 166192b4f..c6a55732d 100644--- a/scrapy/downloadermiddlewares/ajaxcrawl.py+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py@@ -3,10 +3,11 @@ from __future__ import annotationsimport loggingimport refrom typing import TYPE_CHECKING+from warnings import warnfrom w3lib import html-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Responseif TYPE_CHECKING:@@ -30,6 +31,13 @@ class AjaxCrawlMiddleware:if not settings.getbool("AJAXCRAWL_ENABLED"):raise NotConfigured+ warn(+ "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware is deprecated"+ " and will be removed in a future Scrapy version.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+# XXX: Google parses at least first 100k bytes; scrapy's redirect# middleware parses first 4k. 4k turns out to be insufficient# for this middleware, and parsing 100k could be slow.@@ -75,7 +83,6 @@ class AjaxCrawlMiddleware:return _has_ajaxcrawlable_meta(body)-# XXX: move it to w3lib?_ajax_crawlable_re: re.Pattern[str] = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.pyindex 043dc0a12..63bd158f6 100644--- a/tests/test_downloadermiddleware_ajaxcrawlable.py+++ b/tests/test_downloadermiddleware_ajaxcrawlable.py@@ -1,5 +1,7 @@import unittest+import pytest+from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddlewarefrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.spiders import Spider@@ -8,6 +10,7 @@ from scrapy.utils.test import get_crawler__doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"][email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")class AjaxCrawlMiddlewareTest(unittest.TestCase):def setUp(self):crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True})