Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Centralize request fingerprints (#4524)

Adrián Chavesc5627af15bcf
diff --git a/docs/conf.py b/docs/conf.py
index 55aa72d5a..378b01804 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -294,7 +294,9 @@ intersphinx_mapping = {
'tox': ('https://tox.readthedocs.io/en/latest', None),
'twisted': ('https://twistedmatrix.com/documents/current', None),
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
+ 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
}
+intersphinx_disabled_reftypes = []
# Options for sphinx-hoverxref options
diff --git a/docs/news.rst b/docs/news.rst
index 5d92067b5..2d0ab485e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1643,7 +1643,7 @@ New features
:issue:`4370`)
* A new ``keep_fragments`` parameter of
- :func:`scrapy.utils.request.request_fingerprint` allows to generate
+ ``scrapy.utils.request.request_fingerprint`` allows to generate
different fingerprints for requests with different fragments in their URL
(:issue:`4104`)
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 900b19c7a..60b5acd10 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
:class:`scrapy.Spider` subclass and a
:class:`scrapy.settings.Settings` object.
+ .. attribute:: request_fingerprinter
+
+ The request fingerprint builder of this crawler.
+
+ This is used from extensions and middlewares to build short, unique
+ identifiers for requests. See :ref:`request-fingerprints`.
+
.. attribute:: settings
The settings manager of this crawler.
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 391751364..882ff5661 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
:param spider: the spider which was closed
:type spider: :class:`~scrapy.Spider` object
-.. method:: from_crawler(cls, crawler)
+.. classmethod:: from_crawler(cls, crawler)
- If present, this classmethod is called to create a pipeline instance
+ If present, this class method is called to create a pipeline instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the pipeline. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for pipeline to
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 92a471faf..49cb69f67 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -339,6 +339,7 @@ errors if needed::
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
+
.. _errback-cb_kwargs:
Accessing additional data in errback functions
@@ -364,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``::
main_url=failure.request.cb_kwargs['main_url'],
)
+
+.. _request-fingerprints:
+
+Request fingerprints
+--------------------
+
+There are some aspects of scraping, such as filtering out duplicate requests
+(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
+:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
+unique identifier from a :class:`~scrapy.http.Request` object: a request
+fingerprint.
+
+You often do not need to worry about request fingerprints, the default request
+fingerprinter works for most projects.
+
+However, there is no universal way to generate a unique identifier from a
+request, because different situations require comparing requests differently.
+For example, sometimes you may need to compare URLs case-insensitively, include
+URL fragments, exclude certain URL query parameters, include some or all
+headers, etc.
+
+To change how request fingerprints are built for your requests, use the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
+
+.. setting:: REQUEST_FINGERPRINTER_CLASS
+
+REQUEST_FINGERPRINTER_CLASS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+Default: :class:`scrapy.utils.request.RequestFingerprinter`
+
+A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
+import path.
+
+.. autoclass:: scrapy.utils.request.RequestFingerprinter
+
+
+.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
+
+REQUEST_FINGERPRINTER_IMPLEMENTATION
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+Default: ``'PREVIOUS_VERSION'``
+
+Determines which request fingerprinting algorithm is used by the default
+request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
+
+Possible values are:
+
+- ``'PREVIOUS_VERSION'`` (default)
+
+ This implementation uses the same request fingerprinting algorithm as
+ Scrapy PREVIOUS_VERSION and earlier versions.
+
+ Even though this is the default value for backward compatibility reasons,
+ it is a deprecated value.
+
+- ``'VERSION'``
+
+ This implementation was introduced in Scrapy VERSION to fix an issue of the
+ previous implementation.
+
+ New projects should use this value. The :command:`startproject` command
+ sets this value in the generated ``settings.py`` file.
+
+If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
+using Scrapy components where changing the request fingerprinting algorithm
+would cause undesired results, you need to carefully decide when to change the
+value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
+setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
+fingerprinting algorithm and does not log this warning (
+:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
+class).
+
+Scenarios where changing the request fingerprinting algorithm may cause
+undesired results include, for example, using the HTTP cache middleware (see
+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
+Changing the request fingerprinting algorithm would invalidade the current
+cache, requiring you to redownload all requests again.
+
+Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
+your settings to switch already to the request fingerprinting implementation
+that will be the only request fingerprinting implementation available in a
+future version of Scrapy, and remove the deprecation warning triggered by using
+the default value (``'PREVIOUS_VERSION'``).
+
+
+.. _PREVIOUS_VERSION-request-fingerprinter:
+.. _custom-request-fingerprinter:
+
+Writing your own request fingerprinter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A request fingerprinter is a class that must implement the following method:
+
+.. method:: fingerprint(self, request)
+
+ Return a :class:`bytes` object that uniquely identifies *request*.
+
+ See also :ref:`request-fingerprint-restrictions`.
+
+ :param request: request to fingerprint
+ :type request: scrapy.http.Request
+
+Additionally, it may also implement the following methods:
+
+.. classmethod:: from_crawler(cls, crawler)
+
+ If present, this class method is called to create a request fingerprinter
+ instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
+ new instance of the request fingerprinter.
+
+ *crawler* provides access to all Scrapy core components like settings and
+ signals; it is a way for the request fingerprinter to access them and hook
+ its functionality into Scrapy.
+
+ :param crawler: crawler that uses this request fingerprinter
+ :type crawler: :class:`~scrapy.crawler.Crawler` object
+
+.. classmethod:: from_settings(cls, settings)
+
+ If present, and ``from_crawler`` is not defined, this class method is called
+ to create a request fingerprinter instance from a
+ :class:`~scrapy.settings.Settings` object. It must return a new instance of
+ the request fingerprinter.
+
+The ``fingerprint`` method of the default request fingerprinter,
+:class:`scrapy.utils.request.RequestFingerprinter`, uses
+:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
+common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
+in your ``fingerprint`` method implementation:
+
+.. autofunction:: scrapy.utils.request.fingerprint
+
+For example, to take the value of a request header named ``X-ID`` into
+account::
+
+ # my_project/settings.py
+ REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
+
+ # my_project/utils.py
+ from scrapy.utils.request import fingerprint
+
+ class RequestFingerprinter:
+
+ def fingerprint(self, request):
+ return fingerprint(request, include_headers=['X-ID'])
+
+You can also write your own fingerprinting logic from scratch.
+
+However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
+you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
+
+- Caching saves CPU by ensuring that fingerprints are calculated only once
+ per request, and not once per Scrapy component that needs the fingerprint
+ of a request.
+
+- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
+ request objects do not stay in memory forever just because you have
+ references to them in your cache dictionary.
+
+For example, to take into account only the URL of a request, without any prior
+URL canonicalization or taking the request method or body into account::
+
+ from hashlib import sha1
+ from weakref import WeakKeyDictionary
+
+ from scrapy.utils.python import to_bytes
+
+ class RequestFingerprinter:
+
+ cache = WeakKeyDictionary()
+
+ def fingerprint(self, request):
+ if request not in self.cache:
+ fp = sha1()
+ fp.update(to_bytes(request.url))
+ self.cache[request] = fp.digest()
+ return self.cache[request]
+
+If you need to be able to override the request fingerprinting for arbitrary
+requests from your spider callbacks, you may implement a request fingerprinter
+that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
+when available, and then falls back to
+:func:`~scrapy.utils.request.fingerprint`. For example::
+
+ from scrapy.utils.request import fingerprint
+
+ class RequestFingerprinter:
+
+ def fingerprint(self, request):
+ if 'fingerprint' in request.meta:
+ return request.meta['fingerprint']
+ return fingerprint(request)
+
+If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
+without using the deprecated ``'PREVIOUS_VERSION'`` value of the
+:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
+request fingerprinter::
+
+ from hashlib import sha1
+ from weakref import WeakKeyDictionary
+
+ from scrapy.utils.python import to_bytes
+ from w3lib.url import canonicalize_url
+
+ class RequestFingerprinter:
+
+ cache = WeakKeyDictionary()
+
+ def fingerprint(self, request):
+ if request not in self.cache:
+ fp = sha1()
+ fp.update(to_bytes(request.method))
+ fp.update(to_bytes(canonicalize_url(request.url)))
+ fp.update(request.body or b'')
+ self.cache[request] = fp.digest()
+ return self.cache[request]
+
+
+.. _request-fingerprint-restrictions:
+
+Request fingerprint restrictions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Scrapy components that use request fingerprints may impose additional
+restrictions on the format of the fingerprints that your :ref:`request
+fingerprinter <custom-request-fingerprinter>` generates.
+
+The following built-in Scrapy components have such restrictions:
+
+- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
+ value of :setting:`HTTPCACHE_STORAGE`)
+
+ Request fingerprints must be at least 1 byte long.
+
+ Path and filename length limits of the file system of
+ :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
+ the following directory structure is created:
+
+ - :attr:`Spider.name <scrapy.spiders.Spider.name>`
+
+ - first byte of a request fingerprint as hexadecimal
+
+ - fingerprint as hexadecimal
+
+ - filenames up to 16 characters long
+
+ For example, if a request fingerprint is made of 20 bytes (default),
+ :setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
+ and the name of your spider is ``'my_spider'`` your file system must
+ support a file path like::
+
+ /home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
+
+- :class:`scrapy.extensions.httpcache.DbmCacheStorage`
+
+ The underlying DBM implementation must support keys as long as twice
+ the number of bytes of a request fingerprint, plus 5. For example,
+ if a request fingerprint is made of 20 bytes (default),
+ 45-character-long keys must be supported.
+
+
.. _topics-request-meta:
Request.meta special keys
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4e105642d..2046c6446 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -825,12 +825,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
The class used to detect and filter duplicate requests.
-The default (``RFPDupeFilter``) filters based on request fingerprint using
-the ``scrapy.utils.request.request_fingerprint`` function. In order to change
-the way duplicates are checked you could subclass ``RFPDupeFilter`` and
-override its ``request_fingerprint`` method. This method should accept
-scrapy :class:`~scrapy.Request` object and return its fingerprint
-(a string).
+The default (``RFPDupeFilter``) filters based on the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
You can disable filtering of duplicate requests by setting
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a638254f1..fdca7b335 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -51,6 +51,7 @@ class Crawler:
self.spidercls.update_settings(self.settings)
self.signals = SignalManager(self)
+
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
@@ -71,6 +72,12 @@ class Crawler:
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
+ self.request_fingerprinter = create_instance(
+ load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
+ settings=self.settings,
+ crawler=self,
+ )
+
reactor_class = self.settings.get("TWISTED_REACTOR")
if init_reactor:
# this needs to be done after the spider settings are merged,
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 292c68099..d1b0559ef 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,14 +1,16 @@
import logging
import os
from typing import Optional, Set, Type, TypeVar
+from warnings import warn
from twisted.internet.defer import Deferred
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.job import job_dir
-from scrapy.utils.request import referer_str, request_fingerprint
+from scrapy.utils.request import referer_str, RequestFingerprinter
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
@@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
class RFPDupeFilter(BaseDupeFilter):
"""Request Fingerprint duplicates filter"""
- def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
+ def __init__(
+ self,
+ path: Optional[str] = None,
+ debug: bool = False,
+ *,
+ fingerprinter=None,
+ ) -> None:
self.file = None
+ self.fingerprinter = fingerprinter or RequestFingerprinter()
self.fingerprints: Set[str] = set()
self.logdupes = True
self.debug = debug
@@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):
self.fingerprints.update(x.rstrip() for x in self.file)
@classmethod
- def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
+ def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
debug = settings.getbool('DUPEFILTER_DEBUG')
- return cls(job_dir(settings), debug)
+ try:
+ return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
+ except TypeError:
+ warn(
+ "RFPDupeFilter subclasses must either modify their '__init__' "
+ "method to support a 'fingerprinter' parameter or reimplement "
+ "the 'from_settings' class method.",
+ ScrapyDeprecationWarning,
+ )
+ result = cls(job_dir(settings), debug)
+ result.fingerprinter = fingerprinter
+ return result
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ try:
+ return cls.from_settings(
+ crawler.settings,
+ fingerprinter=crawler.request_fingerprinter,
+ )
+ except TypeError:
+ warn(
+ "RFPDupeFilter subclasses must either modify their overridden "
+ "'__init__' method and 'from_settings' class method to "
+ "support a 'fingerprinter' parameter, or reimplement the "
+ "'from_crawler' class method.",
+ ScrapyDeprecationWarning,
+ )
+ result = cls.from_settings(crawler.settings)
+ result.fingerprinter = crawler.request_fingerprinter
+ return result
def request_seen(self, request: Request) -> bool:
fp = self.request_fingerprint(request)
@@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
return False
def request_fingerprint(self, request: Request) -> str:
- return request_fingerprint(request)
+ return self.fingerprinter.fingerprint(request).hex()
def close(self, reason: str) -> None:
if self.file:
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index d0ae29b90..c71484cfa 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.request import request_fingerprint
logger = logging.getLogger(__name__)
@@ -228,6 +227,8 @@ class DbmCacheStorage:
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
+ self._fingerprinter = spider.crawler.request_fingerprinter
+
def close_spider(self, spider):
self.db.close()
@@ -244,7 +245,7 @@ class DbmCacheStorage:
return response
def store_response(self, spider, request, response):
- key = self._request_key(request)
+ key = self._fingerprinter.fingerprint(request).hex()
data = {
'status': response.status,
'url': response.url,
@@ -255,7 +256,7 @@ class DbmCacheStorage:
self.db[f'{key}_time'] = str(time())
def _read_data(self, spider, request):
- key = self._request_key(request)
+ key = self._fingerprinter.fingerprint(request).hex()
db = self.db
tkey = f'{key}_time'
if tkey not in db:
@@ -267,9 +268,6 @@ class DbmCacheStorage:
return pickle.loads(db[f'{key}_data'])
- def _request_key(self, request):
- return request_fingerprint(request)
-
class FilesystemCacheStorage:
@@ -283,6 +281,8 @@ class FilesystemCacheStorage:
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
extra={'spider': spider})
+ self._fingerprinter = spider.crawler.request_fingerprinter
+
def close_spider(self, spider):
pass
@@ -329,7 +329,7 @@ class FilesystemCacheStorage:
f.write(request.body)
def _get_request_path(self, spider, request):
- key = request_fingerprint(request)
+ key = self._fingerprinter.fingerprint(request).hex()
return os.path.join(self.cachedir, spider.name, key[0:2], key)
def _read_meta(self, spider, request):
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 430c37227..5308a9793 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -11,7 +11,6 @@ from scrapy.settings import Settings
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import mustbe_deferred, defer_result
from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.request import request_fingerprint
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.log import failure_to_exc_info
@@ -77,6 +76,7 @@ class MediaPipeline:
except AttributeError:
pipe = cls()
pipe.crawler = crawler
+ pipe._fingerprinter = crawler.request_fingerprinter
return pipe
def open_spider(self, spider):
@@ -90,7 +90,7 @@ class MediaPipeline:
return dfd.addCallback(self.item_completed, item, info)
def _process_request(self, request, info, item):
- fp = request_fingerprint(request)
+ fp = self._fingerprinter.fingerprint(request)
cb = request.callback or (lambda _: _)
eb = request.errback
request.callback = None
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8389a70cb..f5a3efe69 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -246,6 +246,9 @@ REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
+REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
+REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
+
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index a414b5fde..5e541e2c0 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 70ef3ba2b..cf33317ce 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -4,7 +4,9 @@ scrapy.http.Request objects
"""
import hashlib
-from typing import Dict, Iterable, Optional, Tuple, Union
+import json
+import warnings
+from typing import Dict, Iterable, List, Optional, Tuple, Union
from urllib.parse import urlunparse
from weakref import WeakKeyDictionary
@@ -12,13 +14,22 @@ from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
from scrapy import Request, Spider
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes, to_unicode
-_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
-_fingerprint_cache = WeakKeyDictionary()
+_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
+_deprecated_fingerprint_cache = WeakKeyDictionary()
+
+
+def _serialize_headers(headers, request):
+ for header in headers:
+ if header in request.headers:
+ yield header
+ for value in request.headers.getlist(header):
+ yield value
def request_fingerprint(
@@ -27,7 +38,7 @@ def request_fingerprint(
keep_fragments: bool = False,
) -> str:
"""
- Return the request fingerprint.
+ Return the request fingerprint as an hexadecimal string.
The request fingerprint is a hash that uniquely identifies the resource the
request points to. For example, take the following two urls:
@@ -43,7 +54,7 @@ def request_fingerprint(
http://www.example.com/members/offers.html
- Lot of sites use a cookie to store the session id, which adds a random
+ Lots of sites use a cookie to store the session id, which adds a random
component to the HTTP Request and thus should be ignored when calculating
the fingerprint.
@@ -55,29 +66,213 @@ def request_fingerprint(
so they are also ignored by default when calculating the fingerprint.
If you want to include them, set the keep_fragments argument to True
(for instance when handling requests with a headless browser).
-
"""
- headers: Optional[Tuple[bytes, ...]] = None
+ if include_headers or keep_fragments:
+ message = (
+ 'Call to deprecated function '
+ 'scrapy.utils.request.request_fingerprint().\n'
+ '\n'
+ 'If you are using this function in a Scrapy component because you '
+ 'need a non-default fingerprinting algorithm, and you are OK '
+ 'with that non-default fingerprinting algorithm being used by '
+ 'all Scrapy components and not just the one calling this '
+ 'function, use crawler.request_fingerprinter.fingerprint() '
+ 'instead in your Scrapy component (you can get the crawler '
+ 'object from the \'from_crawler\' class method), and use the '
+ '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your '
+ 'non-default fingerprinting algorithm.\n'
+ '\n'
+ 'Otherwise, consider using the '
+ 'scrapy.utils.request.fingerprint() function instead.\n'
+ '\n'
+ 'If you switch to \'fingerprint()\', or assign the '
+ '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses '
+ '\'fingerprint()\', the generated fingerprints will not only be '
+ 'bytes instead of a string, but they will also be different from '
+ 'those generated by \'request_fingerprint()\'. Before you switch, '
+ 'make sure that you understand the consequences of this (e.g. '
+ 'cache invalidation) and are OK with them; otherwise, consider '
+ 'implementing your own function which returns the same '
+ 'fingerprints as the deprecated \'request_fingerprint()\' function.'
+ )
+ else:
+ message = (
+ 'Call to deprecated function '
+ 'scrapy.utils.request.request_fingerprint().\n'
+ '\n'
+ 'If you are using this function in a Scrapy component, and you '
+ 'are OK with users of your component changing the fingerprinting '
+ 'algorithm through settings, use '
+ 'crawler.request_fingerprinter.fingerprint() instead in your '
+ 'Scrapy component (you can get the crawler object from the '
+ '\'from_crawler\' class method).\n'
+ '\n'
+ 'Otherwise, consider using the '
+ 'scrapy.utils.request.fingerprint() function instead.\n'
+ '\n'
+ 'Either way, the resulting fingerprints will be returned as '
+ 'bytes, not as a string, and they will also be different from '
+ 'those generated by \'request_fingerprint()\'. Before you switch, '
+ 'make sure that you understand the consequences of this (e.g. '
+ 'cache invalidation) and are OK with them; otherwise, consider '
+ 'implementing your own function which returns the same '
+ 'fingerprints as the deprecated \'request_fingerprint()\' function.'
+ )
+ warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+ processed_include_headers: Optional[Tuple[bytes, ...]] = None
if include_headers:
- headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
- cache = _fingerprint_cache.setdefault(request, {})
- cache_key = (headers, keep_fragments)
+ processed_include_headers = tuple(
+ to_bytes(h.lower()) for h in sorted(include_headers)
+ )
+ cache = _deprecated_fingerprint_cache.setdefault(request, {})
+ cache_key = (processed_include_headers, keep_fragments)
if cache_key not in cache:
fp = hashlib.sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
fp.update(request.body or b'')
- if headers:
- for hdr in headers:
- if hdr in request.headers:
- fp.update(hdr)
- for v in request.headers.getlist(hdr):
- fp.update(v)
+ if processed_include_headers:
+ for part in _serialize_headers(processed_include_headers, request):
+ fp.update(part)
cache[cache_key] = fp.hexdigest()
return cache[cache_key]
-def request_authenticate(request: Request, username: str, password: str) -> None:
+def _request_fingerprint_as_bytes(*args, **kwargs):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ return bytes.fromhex(request_fingerprint(*args, **kwargs))
+
+
+_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+_fingerprint_cache = WeakKeyDictionary()
+
+
+def fingerprint(
+ request: Request,
+ *,
+ include_headers: Optional[Iterable[Union[bytes, str]]] = None,
+ keep_fragments: bool = False,
+) -> bytes:
+ """
+ Return the request fingerprint.
+
+ The request fingerprint is a hash that uniquely identifies the resource the
+ request points to. For example, take the following two urls:
+
+ http://www.example.com/query?id=111&cat=222
+ http://www.example.com/query?cat=222&id=111
+
+ Even though those are two different URLs both point to the same resource
+ and are equivalent (i.e. they should return the same response).
+
+ Another example are cookies used to store session ids. Suppose the
+ following page is only accessible to authenticated users:
+
+ http://www.example.com/members/offers.html
+
+ Lots of sites use a cookie to store the session id, which adds a random
+ component to the HTTP Request and thus should be ignored when calculating
+ the fingerprint.
+
+ For this reason, request headers are ignored by default when calculating
+ the fingerprint. If you want to include specific headers use the
+ include_headers argument, which is a list of Request headers to include.
+
+ Also, servers usually ignore fragments in urls when handling requests,
+ so they are also ignored by default when calculating the fingerprint.
+ If you want to include them, set the keep_fragments argument to True
+ (for instance when handling requests with a headless browser).
+ """
+ processed_include_headers: Optional[Tuple[bytes, ...]] = None
+ if include_headers:
+ processed_include_headers = tuple(
+ to_bytes(h.lower()) for h in sorted(include_headers)
+ )
+ cache = _fingerprint_cache.setdefault(request, {})
+ cache_key = (processed_include_headers, keep_fragments)
+ if cache_key not in cache:
+ # To decode bytes reliably (JSON does not support bytes), regardless of
+ # character encoding, we use bytes.hex()
+ headers: Dict[str, List[str]] = {}
+ if processed_include_headers:
+ for header in processed_include_headers:
+ if header in request.headers:
+ headers[header.hex()] = [
+ header_value.hex()
+ for header_value in request.headers.getlist(header)
+ ]
+ fingerprint_data = {
+ 'method': to_unicode(request.method),
+ 'url': canonicalize_url(request.url, keep_fragments=keep_fragments),
+ 'body': (request.body or b'').hex(),
+ 'headers': headers,
+ }
+ fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
+ cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
+ return cache[cache_key]
+
+
+class RequestFingerprinter:
+ """Default fingerprinter.
+
+ It takes into account a canonical version
+ (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
+ <scrapy.http.Request.url>` and the values of :attr:`request.method
+ <scrapy.http.Request.method>` and :attr:`request.body
+ <scrapy.http.Request.body>`. It then generates an `SHA1
+ <https://en.wikipedia.org/wiki/SHA-1>`_ hash.
+
+ .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`.
+ """
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+ def __init__(self, crawler=None):
+ if crawler:
+ implementation = crawler.settings.get(
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION'
+ )
+ else:
+ implementation = 'PREVIOUS_VERSION'
+ if implementation == 'PREVIOUS_VERSION':
+ message = (
+ '\'PREVIOUS_VERSION\' is a deprecated value for the '
+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
+ '\n'
+ 'It is also the default value. In other words, it is normal '
+ 'to get this warning if you have not defined a value for the '
+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so '
+ 'for backward compatibility reasons, but it will change in a '
+ 'future version of Scrapy.\n'
+ '\n'
+ 'See the documentation of the '
+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for '
+ 'information on how to handle this deprecation.'
+ )
+ warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+ self._fingerprint = _request_fingerprint_as_bytes
+ elif implementation == 'VERSION':
+ self._fingerprint = fingerprint
+ else:
+ raise ValueError(
+ f'Got an invalid value on setting '
+ f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
+ f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '
+ f'and \'VERSION\'.'
+ )
+
+ def fingerprint(self, request):
+ return self._fingerprint(request)
+
+
+def request_authenticate(
+ request: Request,
+ username: str,
+ password: str,
+) -> None:
"""Authenticate the given request (in place) using the HTTP basic access
authentication mechanism (RFC 2617) and the given username and password
"""
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 24c38283a..b90ea5009 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -54,7 +54,7 @@ def get_ftp_content_and_delete(
return "".join(ftp_data)
-def get_crawler(spidercls=None, settings_dict=None):
+def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
priority.
@@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None):
from scrapy.crawler import CrawlerRunner
from scrapy.spiders import Spider
- runner = CrawlerRunner(settings_dict)
+ # Set by default settings that prevent deprecation warnings.
+ settings = {}
+ if prevent_warnings:
+ settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION'
+ settings.update(settings_dict or {})
+ runner = CrawlerRunner(settings)
return runner.create_crawler(spidercls or Spider)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8f6227109..f7aa769e4 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -104,7 +104,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):
custom_settings = {
'LOG_LEVEL': 'INFO',
'LOG_FILE': log_file,
- # disable telnet if not available to avoid an extra warning
+ # settings to avoid extra warnings
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
}
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 680bb6dc8..b7df2554a 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -15,6 +15,16 @@ from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
+def _get_dupefilter(*, crawler=None, settings=None, open=True):
+ if crawler is None:
+ crawler = get_crawler(settings_dict=settings)
+ scheduler = Scheduler.from_crawler(crawler)
+ dupefilter = scheduler.df
+ if open:
+ dupefilter.open()
+ return dupefilter
+
+
class FromCrawlerRFPDupeFilter(RFPDupeFilter):
@classmethod
@@ -64,9 +74,7 @@ class RFPDupeFilterTest(unittest.TestCase):
self.assertEqual(scheduler.df.method, 'n/a')
def test_filter(self):
- dupefilter = RFPDupeFilter()
- dupefilter.open()
-
+ dupefilter = _get_dupefilter()
r1 = Request('http://scrapytest.org/1')
r2 = Request('http://scrapytest.org/2')
r3 = Request('http://scrapytest.org/2')
@@ -85,7 +93,7 @@ class RFPDupeFilterTest(unittest.TestCase):
path = tempfile.mkdtemp()
try:
- df = RFPDupeFilter(path)
+ df = _get_dupefilter(settings={'JOBDIR': path}, open=False)
try:
df.open()
assert not df.request_seen(r1)
@@ -93,7 +101,8 @@ class RFPDupeFilterTest(unittest.TestCase):
finally:
df.close('finished')
- df2 = RFPDupeFilter(path)
+ df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False)
+ assert df != df2
try:
df2.open()
assert df2.request_seen(r1)
@@ -109,26 +118,24 @@ class RFPDupeFilterTest(unittest.TestCase):
output of request_seen.
"""
+ dupefilter = _get_dupefilter()
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/INDEX.html')
- dupefilter = RFPDupeFilter()
- dupefilter.open()
-
assert not dupefilter.request_seen(r1)
assert not dupefilter.request_seen(r2)
dupefilter.close('finished')
- class CaseInsensitiveRFPDupeFilter(RFPDupeFilter):
+ class RequestFingerprinter:
- def request_fingerprint(self, request):
+ def fingerprint(self, request):
fp = hashlib.sha1()
fp.update(to_bytes(request.url.lower()))
- return fp.hexdigest()
+ return fp.digest()
- case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter()
- case_insensitive_dupefilter.open()
+ settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter}
+ case_insensitive_dupefilter = _get_dupefilter(settings=settings)
assert not case_insensitive_dupefilter.request_seen(r1)
assert case_insensitive_dupefilter.request_seen(r2)
@@ -142,8 +149,10 @@ class RFPDupeFilterTest(unittest.TestCase):
r1 = Request('http://scrapytest.org/1')
path = tempfile.mkdtemp()
+ crawler = get_crawler(settings_dict={'JOBDIR': path})
try:
- df = RFPDupeFilter(path)
+ scheduler = Scheduler.from_crawler(crawler)
+ df = scheduler.df
df.open()
df.request_seen(r1)
df.close('finished')
@@ -164,11 +173,8 @@ class RFPDupeFilterTest(unittest.TestCase):
settings = {'DUPEFILTER_DEBUG': False,
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
- scheduler = Scheduler.from_crawler(crawler)
spider = SimpleSpider.from_crawler(crawler)
-
- dupefilter = scheduler.df
- dupefilter.open()
+ dupefilter = _get_dupefilter(crawler=crawler)
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html')
@@ -193,11 +199,41 @@ class RFPDupeFilterTest(unittest.TestCase):
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
- scheduler = Scheduler.from_crawler(crawler)
spider = SimpleSpider.from_crawler(crawler)
+ dupefilter = _get_dupefilter(crawler=crawler)
+
+ r1 = Request('http://scrapytest.org/index.html')
+ r2 = Request('http://scrapytest.org/index.html',
+ headers={'Referer': 'http://scrapytest.org/INDEX.html'})
- dupefilter = scheduler.df
- dupefilter.open()
+ dupefilter.log(r1, spider)
+ dupefilter.log(r2, spider)
+
+ assert crawler.stats.get_value('dupefilter/filtered') == 2
+ log.check_present(
+ (
+ 'scrapy.dupefilters',
+ 'DEBUG',
+ 'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
+ )
+ )
+ log.check_present(
+ (
+ 'scrapy.dupefilters',
+ 'DEBUG',
+ 'Filtered duplicate request: <GET http://scrapytest.org/index.html>'
+ ' (referer: http://scrapytest.org/INDEX.html)'
+ )
+ )
+
+ dupefilter.close('finished')
+
+ def test_log_debug_default_dupefilter(self):
+ with LogCapture() as log:
+ settings = {'DUPEFILTER_DEBUG': True}
+ crawler = get_crawler(SimpleSpider, settings_dict=settings)
+ spider = SimpleSpider.from_crawler(crawler)
+ dupefilter = _get_dupefilter(crawler=crawler)
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html',
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 0ff2045ed..4228173ed 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -25,6 +25,7 @@ from scrapy.pipelines.files import (
from scrapy.settings import Settings
from scrapy.utils.test import (
assert_gcs_environ,
+ get_crawler,
get_ftp_content_and_delete,
get_gcs_content_and_delete,
skip_if_no_boto,
@@ -47,7 +48,9 @@ class FilesPipelineTestCase(unittest.TestCase):
def setUp(self):
self.tempdir = mkdtemp()
- self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir}))
+ settings_dict = {'FILES_STORE': self.tempdir}
+ crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
+ self.pipeline = FilesPipeline.from_crawler(crawler)
self.pipeline.download_func = _mocked_download_func
self.pipeline.open_spider(None)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index a802c7cf1..84e867660 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -7,17 +7,17 @@ from twisted.python.failure import Failure
from twisted.internet import reactor
from twisted.internet.defer import Deferred, inlineCallbacks
+from scrapy import signals
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.request import request_fingerprint
+from scrapy.pipelines.files import FileException
from scrapy.pipelines.images import ImagesPipeline
from scrapy.pipelines.media import MediaPipeline
-from scrapy.pipelines.files import FileException
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all
-from scrapy import signals
+from scrapy.utils.test import get_crawler
try:
@@ -39,11 +39,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
settings = None
def setUp(self):
- self.spider = Spider('media.com')
- self.pipe = self.pipeline_class(download_func=_mocked_download_func,
- settings=Settings(self.settings))
+ spider_cls = Spider
+ self.spider = spider_cls('media.com')
+ crawler = get_crawler(spider_cls, self.settings)
+ self.pipe = self.pipeline_class.from_crawler(crawler)
+ self.pipe.download_func = _mocked_download_func
self.pipe.open_spider(self.spider)
self.info = self.pipe.spiderinfo
+ self.fingerprint = crawler.request_fingerprinter.fingerprint
def tearDown(self):
for name, signal in vars(signals).items():
@@ -156,7 +159,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
self.assertEqual(failure.value.__context__, def_gen_return_exc)
# Let's calculate the request fingerprint and fake some runtime data...
- fp = request_fingerprint(request)
+ fp = self.fingerprint(request)
info = self.pipe.spiderinfo
info.downloading.add(fp)
info.waiting[fp] = []
@@ -273,7 +276,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req) # pass a single item
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
- assert request_fingerprint(req) in self.info.downloaded
+ self.assertIn(self.fingerprint(req), self.info.downloaded)
# returns iterable of Requests
req1 = Request('http://url1')
@@ -281,8 +284,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=iter([req1, req2]))
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
- assert request_fingerprint(req1) in self.info.downloaded
- assert request_fingerprint(req2) in self.info.downloaded
+ assert self.fingerprint(req1) in self.info.downloaded
+ assert self.fingerprint(req2) in self.info.downloaded
@inlineCallbacks
def test_results_are_cached_across_multiple_items(self):
@@ -298,7 +301,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req2)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertTrue(new_item is item)
- self.assertEqual(request_fingerprint(req1), request_fingerprint(req2))
+ self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))
self.assertEqual(new_item['results'], [(True, rsp1)])
@inlineCallbacks
@@ -314,7 +317,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
@inlineCallbacks
def test_wait_if_request_is_downloading(self):
def _check_downloading(response):
- fp = request_fingerprint(req1)
+ fp = self.fingerprint(req1)
self.assertTrue(fp in self.info.downloading)
self.assertTrue(fp in self.info.waiting)
self.assertTrue(fp not in self.info.downloaded)
@@ -351,7 +354,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
def __init__(self, *args, **kwargs):
- super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs)
+ super().__init__(*args, **kwargs)
self._mockcalled = []
def get_media_requests(self, item, info):
@@ -369,19 +372,19 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
def media_to_download(self, request, info):
self._mockcalled.append('media_to_download')
- return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info)
+ return super().media_to_download(request, info)
def media_downloaded(self, response, request, info):
self._mockcalled.append('media_downloaded')
- return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info)
+ return super().media_downloaded(response, request, info)
def file_downloaded(self, response, request, info):
self._mockcalled.append('file_downloaded')
- return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info)
+ return super().file_downloaded(response, request, info)
def file_path(self, request, response=None, info=None):
self._mockcalled.append('file_path')
- return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)
+ return super().file_path(request, response, info)
def thumb_path(self, request, thumb_id, response=None, info=None):
self._mockcalled.append('thumb_path')
@@ -393,18 +396,20 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
def image_downloaded(self, response, request, info):
self._mockcalled.append('image_downloaded')
- return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info)
+ return super().image_downloaded(response, request, info)
class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
skip = skip_pillow
def setUp(self):
- self.pipe = MockedMediaPipelineDeprecatedMethods(
- store_uri='store-uri',
- download_func=_mocked_download_func,
- settings=Settings({"IMAGES_THUMBS": {'small': (50, 50)}})
- )
+ settings_dict = {
+ 'IMAGES_STORE': 'store-uri',
+ 'IMAGES_THUMBS': {'small': (50, 50)},
+ }
+ crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
+ self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
+ self.pipe.download_func = _mocked_download_func
self.pipe.open_spider(None)
self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 7e0049b1d..e9edfee98 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,33 +1,166 @@
import unittest
+import warnings
+from hashlib import sha1
+from typing import Dict, Mapping, Optional, Tuple, Union
+from weakref import WeakKeyDictionary
+
+import pytest
+from w3lib.url import canonicalize_url
+
from scrapy.http import Request
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.python import to_bytes
from scrapy.utils.request import (
+ _deprecated_fingerprint_cache,
_fingerprint_cache,
+ _request_fingerprint_as_bytes,
+ fingerprint,
request_authenticate,
request_fingerprint,
request_httprepr,
)
+from scrapy.utils.test import get_crawler
class UtilsRequestTest(unittest.TestCase):
- def test_request_fingerprint(self):
+ def test_request_authenticate(self):
+ r = Request("http://www.example.com")
+ request_authenticate(r, 'someuser', 'somepass')
+ self.assertEqual(r.headers['Authorization'], b'Basic c29tZXVzZXI6c29tZXBhc3M=')
+
+ def test_request_httprepr(self):
+ r1 = Request("http://www.example.com")
+ self.assertEqual(request_httprepr(r1), b'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
+
+ r1 = Request("http://www.example.com/some/page.html?arg=1")
+ self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
+
+ r1 = Request("http://www.example.com", method='POST',
+ headers={"Content-type": b"text/html"}, body=b"Some body")
+ self.assertEqual(
+ request_httprepr(r1),
+ b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body'
+ )
+
+ def test_request_httprepr_for_non_http_request(self):
+ # the representation is not important but it must not fail.
+ request_httprepr(Request("file:///tmp/foo.txt"))
+ request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
+
+
+class FingerprintTest(unittest.TestCase):
+ maxDiff = None
+
+ function = staticmethod(fingerprint)
+ cache: Union[
+ "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]",
+ "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]",
+ ] = _fingerprint_cache
+ default_cache_key = (None, False)
+ known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
+ (
+ Request("http://example.org"),
+ b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD',
+ {},
+ ),
+ (
+ Request("https://example.org"),
+ b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+ {},
+ ),
+ (
+ Request("https://example.org?a"),
+ b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b"),
+ b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b&a"),
+ b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b&a=c"),
+ b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef',
+ {},
+ ),
+ (
+ Request("https://example.org", method='POST'),
+ b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d',
+ {},
+ ),
+ (
+ Request("https://example.org", body=b'a'),
+ b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_',
+ {},
+ ),
+ (
+ Request("https://example.org", method='POST', body=b'a'),
+ b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7',
+ {},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+ {},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.',
+ {'include_headers': ['A']},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f',
+ {'keep_fragments': True},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11',
+ {'include_headers': ['A'], 'keep_fragments': True},
+ ),
+ (
+ Request("https://example.org/ab"),
+ b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6',
+ {},
+ ),
+ (
+ Request("https://example.org/a", body=b'b'),
+ b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5',
+ {},
+ ),
+ )
+
+ def test_query_string_key_order(self):
r1 = Request("http://www.example.com/query?id=111&cat=222")
r2 = Request("http://www.example.com/query?cat=222&id=111")
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
+ self.assertEqual(self.function(r1), self.function(r1))
+ self.assertEqual(self.function(r1), self.function(r2))
+ def test_query_string_key_without_value(self):
r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
- self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
+ self.assertNotEqual(self.function(r1), self.function(r2))
- # make sure caching is working
- self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)])
+ def test_caching(self):
+ r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+ self.assertEqual(
+ self.function(r1),
+ self.cache[r1][self.default_cache_key]
+ )
+ def test_header(self):
r1 = Request("http://www.example.com/members/offers.html")
r2 = Request("http://www.example.com/members/offers.html")
r2.headers['SESSIONID'] = b"somehash"
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
+ self.assertEqual(self.function(r1), self.function(r2))
+ def test_headers(self):
r1 = Request("http://www.example.com/")
r2 = Request("http://www.example.com/")
r2.headers['Accept-Language'] = b'en'
@@ -35,62 +168,512 @@ class UtilsRequestTest(unittest.TestCase):
r3.headers['Accept-Language'] = b'en'
r3.headers['SESSIONID'] = b"somehash"
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3))
+ self.assertEqual(self.function(r1), self.function(r2), self.function(r3))
- self.assertEqual(request_fingerprint(r1),
- request_fingerprint(r1, include_headers=['Accept-Language']))
+ self.assertEqual(self.function(r1),
+ self.function(r1, include_headers=['Accept-Language']))
self.assertNotEqual(
- request_fingerprint(r1),
- request_fingerprint(r2, include_headers=['Accept-Language']))
+ self.function(r1),
+ self.function(r2, include_headers=['Accept-Language']))
- self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
- request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
+ self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']),
+ self.function(r3, include_headers=['SESSIONID', 'Accept-Language']))
+ def test_fragment(self):
r1 = Request("http://www.example.com/test.html")
r2 = Request("http://www.example.com/test.html#fragment")
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True))
- self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True))
- self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True))
+ self.assertEqual(self.function(r1), self.function(r2))
+ self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True))
+ self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True))
+ self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True))
+ def test_method_and_body(self):
r1 = Request("http://www.example.com")
r2 = Request("http://www.example.com", method='POST')
r3 = Request("http://www.example.com", method='POST', body=b'request body')
- self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
- self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3))
+ self.assertNotEqual(self.function(r1), self.function(r2))
+ self.assertNotEqual(self.function(r2), self.function(r3))
+ def test_request_replace(self):
# cached fingerprint must be cleared on request copy
r1 = Request("http://www.example.com")
- fp1 = request_fingerprint(r1)
+ fp1 = self.function(r1)
r2 = r1.replace(url="http://www.example.com/other")
- fp2 = request_fingerprint(r2)
+ fp2 = self.function(r2)
self.assertNotEqual(fp1, fp2)
- def test_request_authenticate(self):
- r = Request("http://www.example.com")
- request_authenticate(r, 'someuser', 'somepass')
- self.assertEqual(r.headers['Authorization'], b'Basic c29tZXVzZXI6c29tZXBhc3M=')
+ def test_part_separation(self):
+ # An old implementation used to serialize request data in a way that
+ # would put the body right after the URL.
+ r1 = Request("http://www.example.com/foo")
+ fp1 = self.function(r1)
+ r2 = Request("http://www.example.com/f", body=b'oo')
+ fp2 = self.function(r2)
+ self.assertNotEqual(fp1, fp2)
- def test_request_httprepr(self):
- r1 = Request("http://www.example.com")
- self.assertEqual(request_httprepr(r1), b'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
+ def test_hashes(self):
+ """Test hardcoded hashes, to make sure future changes to not introduce
+ backward incompatibilities."""
+ actual = [
+ self.function(request, **kwargs)
+ for request, _, kwargs in self.known_hashes
+ ]
+ expected = [
+ _fingerprint
+ for _, _fingerprint, _ in self.known_hashes
+ ]
+ self.assertEqual(actual, expected)
- r1 = Request("http://www.example.com/some/page.html?arg=1")
- self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
- r1 = Request("http://www.example.com", method='POST',
- headers={"Content-type": b"text/html"}, body=b"Some body")
+class RequestFingerprintTest(FingerprintTest):
+ function = staticmethod(request_fingerprint)
+ cache = _deprecated_fingerprint_cache
+ known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
+ (
+ Request("http://example.org"),
+ 'b2e5245ef826fd9576c93bd6e392fce3133fab62',
+ {},
+ ),
+ (
+ Request("https://example.org"),
+ 'bd10a0a89ea32cdee77917320f1309b0da87e892',
+ {},
+ ),
+ (
+ Request("https://example.org?a"),
+ '2fb7d48ae02f04b749f40caa969c0bc3c43204ce',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b"),
+ '42e5fe149b147476e3f67ad0670c57b4cc57856a',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b&a"),
+ 'd23a9787cb56c6375c2cae4453c5a8c634526942',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b&a=c"),
+ '9a18a7a8552a9182b7f1e05d33876409e421e5c5',
+ {},
+ ),
+ (
+ Request("https://example.org", method='POST'),
+ 'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6',
+ {},
+ ),
+ (
+ Request("https://example.org", body=b'a'),
+ '4bb136e54e715a4ea7a9dd1101831765d33f2d60',
+ {},
+ ),
+ (
+ Request("https://example.org", method='POST', body=b'a'),
+ '6c6595374a304b293be762f7b7be3f54e9947c65',
+ {},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ 'bd10a0a89ea32cdee77917320f1309b0da87e892',
+ {},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ '515b633cb3ca502a33a9d8c890e889ec1e425e65',
+ {'include_headers': ['A']},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ '505c96e7da675920dfef58725e8c957dfdb38f47',
+ {'keep_fragments': True},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ 'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da',
+ {'include_headers': ['A'], 'keep_fragments': True},
+ ),
+ (
+ Request("https://example.org/ab"),
+ '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+ {},
+ ),
+ (
+ Request("https://example.org/a", body=b'b'),
+ '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+ {},
+ ),
+ )
+
+ @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+ def test_part_separation(self):
+ super().test_part_separation()
+
+ def test_deprecation_default_parameters(self):
+ with pytest.warns(ScrapyDeprecationWarning) as warnings:
+ self.function(Request("http://www.example.com"))
+ messages = [str(warning.message) for warning in warnings]
+ self.assertTrue(
+ any(
+ 'Call to deprecated function' in message
+ for message in messages
+ )
+ )
+ self.assertFalse(any('non-default' in message for message in messages))
+
+ def test_deprecation_non_default_parameters(self):
+ with pytest.warns(ScrapyDeprecationWarning) as warnings:
+ self.function(Request("http://www.example.com"), keep_fragments=True)
+ messages = [str(warning.message) for warning in warnings]
+ self.assertTrue(
+ any(
+ 'Call to deprecated function' in message
+ for message in messages
+ )
+ )
+ self.assertTrue(any('non-default' in message for message in messages))
+
+
+class RequestFingerprintAsBytesTest(FingerprintTest):
+ function = staticmethod(_request_fingerprint_as_bytes)
+ cache = _deprecated_fingerprint_cache
+ known_hashes = RequestFingerprintTest.known_hashes
+
+ def test_caching(self):
+ r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
self.assertEqual(
- request_httprepr(r1),
- b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body'
+ self.function(r1),
+ bytes.fromhex(self.cache[r1][self.default_cache_key])
)
- def test_request_httprepr_for_non_http_request(self):
- # the representation is not important but it must not fail.
- request_httprepr(Request("file:///tmp/foo.txt"))
- request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
+ @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+ def test_part_separation(self):
+ super().test_part_separation()
+
+ def test_hashes(self):
+ actual = [
+ self.function(request, **kwargs)
+ for request, _, kwargs in self.known_hashes
+ ]
+ expected = [
+ bytes.fromhex(_fingerprint)
+ for _, _fingerprint, _ in self.known_hashes
+ ]
+ self.assertEqual(actual, expected)
+
+
+_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
+
+
+def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
+ if include_headers:
+ include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+ cache = _fingerprint_cache_2_6.setdefault(request, {})
+ cache_key = (include_headers, keep_fragments)
+ if cache_key not in cache:
+ fp = sha1()
+ fp.update(to_bytes(request.method))
+ fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
+ fp.update(request.body or b'')
+ if include_headers:
+ for hdr in include_headers:
+ if hdr in request.headers:
+ fp.update(hdr)
+ for v in request.headers.getlist(hdr):
+ fp.update(v)
+ cache[cache_key] = fp.hexdigest()
+ return cache[cache_key]
+
+
+REQUEST_OBJECTS_TO_TEST = (
+ Request("http://www.example.com/"),
+ Request("http://www.example.com/query?id=111&cat=222"),
+ Request("http://www.example.com/query?cat=222&id=111"),
+ Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'),
+ Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'),
+ Request("http://www.example.com/members/offers.html"),
+ Request(
+ "http://www.example.com/members/offers.html",
+ headers={'SESSIONID': b"somehash"},
+ ),
+ Request(
+ "http://www.example.com/",
+ headers={'Accept-Language': b"en"},
+ ),
+ Request(
+ "http://www.example.com/",
+ headers={
+ 'Accept-Language': b"en",
+ 'SESSIONID': b"somehash",
+ },
+ ),
+ Request("http://www.example.com/test.html"),
+ Request("http://www.example.com/test.html#fragment"),
+ Request("http://www.example.com", method='POST'),
+ Request("http://www.example.com", method='POST', body=b'request body'),
+)
+
+
+class BackwardCompatibilityTestCase(unittest.TestCase):
+
+ def test_function_backward_compatibility(self):
+ include_headers_to_test = (
+ None,
+ ['Accept-Language'],
+ ['accept-language', 'sessionid'],
+ ['SESSIONID', 'Accept-Language'],
+ )
+ for request_object in REQUEST_OBJECTS_TO_TEST:
+ for include_headers in include_headers_to_test:
+ for keep_fragments in (False, True):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ fp = request_fingerprint(
+ request_object,
+ include_headers=include_headers,
+ keep_fragments=keep_fragments,
+ )
+ old_fp = request_fingerprint_2_6(
+ request_object,
+ include_headers=include_headers,
+ keep_fragments=keep_fragments,
+ )
+ self.assertEqual(fp, old_fp)
+
+ def test_component_backward_compatibility(self):
+ for request_object in REQUEST_OBJECTS_TO_TEST:
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ crawler = get_crawler(prevent_warnings=False)
+ fp = crawler.request_fingerprinter.fingerprint(request_object)
+ old_fp = request_fingerprint_2_6(request_object)
+ self.assertEqual(fp.hex(), old_fp)
+
+ def test_custom_component_backward_compatibility(self):
+ """Tests that the backward-compatible request fingerprinting class featured
+ in the documentation is indeed backward compatible and does not cause a
+ warning to be logged."""
+
+ class RequestFingerprinter:
+
+ cache = WeakKeyDictionary()
+
+ def fingerprint(self, request):
+ if request not in self.cache:
+ fp = sha1()
+ fp.update(to_bytes(request.method))
+ fp.update(to_bytes(canonicalize_url(request.url)))
+ fp.update(request.body or b'')
+ self.cache[request] = fp.digest()
+ return self.cache[request]
+
+ for request_object in REQUEST_OBJECTS_TO_TEST:
+ with warnings.catch_warnings() as logged_warnings:
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ }
+ crawler = get_crawler(settings_dict=settings)
+ fp = crawler.request_fingerprinter.fingerprint(request_object)
+ old_fp = request_fingerprint_2_6(request_object)
+ self.assertEqual(fp.hex(), old_fp)
+ self.assertFalse(logged_warnings)
+
+
+class RequestFingerprinterTestCase(unittest.TestCase):
+
+ def test_default_implementation(self):
+ with warnings.catch_warnings(record=True) as logged_warnings:
+ crawler = get_crawler(prevent_warnings=False)
+ request = Request('https://example.com')
+ self.assertEqual(
+ crawler.request_fingerprinter.fingerprint(request),
+ _request_fingerprint_as_bytes(request),
+ )
+ self.assertTrue(logged_warnings)
+
+ def test_deprecated_implementation(self):
+ settings = {
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION',
+ }
+ with warnings.catch_warnings(record=True) as logged_warnings:
+ crawler = get_crawler(settings_dict=settings)
+ request = Request('https://example.com')
+ self.assertEqual(
+ crawler.request_fingerprinter.fingerprint(request),
+ _request_fingerprint_as_bytes(request),
+ )
+ self.assertTrue(logged_warnings)
+
+ def test_recommended_implementation(self):
+ settings = {
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+ }
+ with warnings.catch_warnings(record=True) as logged_warnings:
+ crawler = get_crawler(settings_dict=settings)
+ request = Request('https://example.com')
+ self.assertEqual(
+ crawler.request_fingerprinter.fingerprint(request),
+ fingerprint(request),
+ )
+ self.assertFalse(logged_warnings)
+
+ def test_unknown_implementation(self):
+ settings = {
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5',
+ }
+ with self.assertRaises(ValueError):
+ get_crawler(settings_dict=settings)
+
+
+class CustomRequestFingerprinterTestCase(unittest.TestCase):
+
+ def test_include_headers(self):
+
+ class RequestFingerprinter:
+
+ def fingerprint(self, request):
+ return fingerprint(request, include_headers=['X-ID'])
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ r1 = Request("http://www.example.com", headers={'X-ID': '1'})
+ fp1 = crawler.request_fingerprinter.fingerprint(r1)
+ r2 = Request("http://www.example.com", headers={'X-ID': '2'})
+ fp2 = crawler.request_fingerprinter.fingerprint(r2)
+ self.assertNotEqual(fp1, fp2)
+
+ def test_dont_canonicalize(self):
+
+ class RequestFingerprinter:
+ cache = WeakKeyDictionary()
+
+ def fingerprint(self, request):
+ if request not in self.cache:
+ fp = sha1()
+ fp.update(to_bytes(request.url))
+ self.cache[request] = fp.digest()
+ return self.cache[request]
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ r1 = Request("http://www.example.com?a=1&a=2")
+ fp1 = crawler.request_fingerprinter.fingerprint(r1)
+ r2 = Request("http://www.example.com?a=2&a=1")
+ fp2 = crawler.request_fingerprinter.fingerprint(r2)
+ self.assertNotEqual(fp1, fp2)
+
+ def test_meta(self):
+
+ class RequestFingerprinter:
+
+ def fingerprint(self, request):
+ if 'fingerprint' in request.meta:
+ return request.meta['fingerprint']
+ return fingerprint(request)
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ r1 = Request("http://www.example.com")
+ fp1 = crawler.request_fingerprinter.fingerprint(r1)
+ r2 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+ fp2 = crawler.request_fingerprinter.fingerprint(r2)
+ r3 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+ fp3 = crawler.request_fingerprinter.fingerprint(r3)
+ r4 = Request("http://www.example.com", meta={'fingerprint': 'b'})
+ fp4 = crawler.request_fingerprinter.fingerprint(r4)
+ self.assertNotEqual(fp1, fp2)
+ self.assertNotEqual(fp1, fp4)
+ self.assertNotEqual(fp2, fp4)
+ self.assertEqual(fp2, fp3)
+
+ def test_from_crawler(self):
+
+ class RequestFingerprinter:
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+ def __init__(self, crawler):
+ self._fingerprint = crawler.settings['FINGERPRINT']
+
+ def fingerprint(self, request):
+ return self._fingerprint
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ 'FINGERPRINT': b'fingerprint',
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ request = Request("http://www.example.com")
+ fingerprint = crawler.request_fingerprinter.fingerprint(request)
+ self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+ def test_from_settings(self):
+
+ class RequestFingerprinter:
+
+ @classmethod
+ def from_settings(cls, settings):
+ return cls(settings)
+
+ def __init__(self, settings):
+ self._fingerprint = settings['FINGERPRINT']
+
+ def fingerprint(self, request):
+ return self._fingerprint
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ 'FINGERPRINT': b'fingerprint',
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ request = Request("http://www.example.com")
+ fingerprint = crawler.request_fingerprinter.fingerprint(request)
+ self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+ def test_from_crawler_and_settings(self):
+
+ class RequestFingerprinter:
+
+ # This method is ignored due to the presence of from_crawler
+ @classmethod
+ def from_settings(cls, settings):
+ return cls(settings)
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+ def __init__(self, crawler):
+ self._fingerprint = crawler.settings['FINGERPRINT']
+
+ def fingerprint(self, request):
+ return self._fingerprint
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ 'FINGERPRINT': b'fingerprint',
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ request = Request("http://www.example.com")
+ fingerprint = crawler.request_fingerprinter.fingerprint(request)
+ self.assertEqual(fingerprint, settings['FINGERPRINT'])
if __name__ == "__main__":