Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve dont_filter documentation (#7245)
Varun Chawla0cfc4e4386c6
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex d306f1f6e..61e50927d 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -210,8 +210,22 @@ class Request(object_ref):#: default), or those components should not filter out this request#: (``True``).#:- #: This attribute is commonly set to ``True`` to prevent duplicate- #: requests from being filtered out.+ #: The following built-in components check this attribute:+ #:+ #: - The :ref:`scheduler <topics-scheduler>` uses it to skip+ #: duplicate request filtering (see+ #: :setting:`DUPEFILTER_CLASS`). When set to ``True``, the+ #: request is not checked against the duplicate filter,+ #: allowing requests that would otherwise be considered duplicates+ #: to be scheduled multiple times.+ #: - :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`+ #: uses it to allow requests to domains not in+ #: :attr:`~scrapy.Spider.allowed_domains`. To skip only the offsite+ #: filter without affecting other components, consider using the+ #: :reqmeta:`allow_offsite` request meta key instead.+ #:+ #: Third-party components may also use this attribute to decide whether+ #: to filter out a request.#:#: When defining the start URLs of a spider through#: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by