Scrapy, a fast high-level web crawling & scraping framework for Python.
Centralize request fingerprints (#4524)
Adrián Chavesc5627af15bcf
diff --git a/docs/conf.py b/docs/conf.pyindex 55aa72d5a..378b01804 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -294,7 +294,9 @@ intersphinx_mapping = {'tox': ('https://tox.readthedocs.io/en/latest', None),'twisted': ('https://twistedmatrix.com/documents/current', None),'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),+ 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),}+intersphinx_disabled_reftypes = []# Options for sphinx-hoverxref optionsdiff --git a/docs/news.rst b/docs/news.rstindex 5d92067b5..2d0ab485e 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -1643,7 +1643,7 @@ New features:issue:`4370`)* A new ``keep_fragments`` parameter of- :func:`scrapy.utils.request.request_fingerprint` allows to generate+ ``scrapy.utils.request.request_fingerprint`` allows to generatedifferent fingerprints for requests with different fragments in their URL(:issue:`4104`)diff --git a/docs/topics/api.rst b/docs/topics/api.rstindex 900b19c7a..60b5acd10 100644--- a/docs/topics/api.rst+++ b/docs/topics/api.rst@@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares:class:`scrapy.Spider` subclass and a:class:`scrapy.settings.Settings` object.+ .. attribute:: request_fingerprinter++ The request fingerprint builder of this crawler.++ This is used from extensions and middlewares to build short, unique+ identifiers for requests. See :ref:`request-fingerprints`.+.. attribute:: settingsThe settings manager of this crawler.diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rstindex 391751364..882ff5661 100644--- a/docs/topics/item-pipeline.rst+++ b/docs/topics/item-pipeline.rst@@ -60,9 +60,9 @@ Additionally, they may also implement the following methods::param spider: the spider which was closed:type spider: :class:`~scrapy.Spider` object-.. method:: from_crawler(cls, crawler)+.. classmethod:: from_crawler(cls, crawler)- If present, this classmethod is called to create a pipeline instance+ If present, this class method is called to create a pipeline instancefrom a :class:`~scrapy.crawler.Crawler`. It must return a new instanceof the pipeline. Crawler object provides access to all Scrapy corecomponents like settings and signals; it is a way for pipeline todiff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 92a471faf..49cb69f67 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -339,6 +339,7 @@ errors if needed::request = failure.requestself.logger.error('TimeoutError on %s', request.url)+.. _errback-cb_kwargs:Accessing additional data in errback functions@@ -364,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``::main_url=failure.request.cb_kwargs['main_url'],)++.. _request-fingerprints:++Request fingerprints+--------------------++There are some aspects of scraping, such as filtering out duplicate requests+(see :setting:`DUPEFILTER_CLASS`) or caching responses (see+:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,+unique identifier from a :class:`~scrapy.http.Request` object: a request+fingerprint.++You often do not need to worry about request fingerprints, the default request+fingerprinter works for most projects.++However, there is no universal way to generate a unique identifier from a+request, because different situations require comparing requests differently.+For example, sometimes you may need to compare URLs case-insensitively, include+URL fragments, exclude certain URL query parameters, include some or all+headers, etc.++To change how request fingerprints are built for your requests, use the+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.++.. setting:: REQUEST_FINGERPRINTER_CLASS++REQUEST_FINGERPRINTER_CLASS+~~~~~~~~~~~~~~~~~~~~~~~~~~~++.. versionadded:: VERSION++Default: :class:`scrapy.utils.request.RequestFingerprinter`++A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its+import path.++.. autoclass:: scrapy.utils.request.RequestFingerprinter+++.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION++REQUEST_FINGERPRINTER_IMPLEMENTATION+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++.. versionadded:: VERSION++Default: ``'PREVIOUS_VERSION'``++Determines which request fingerprinting algorithm is used by the default+request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).++Possible values are:++- ``'PREVIOUS_VERSION'`` (default)++ This implementation uses the same request fingerprinting algorithm as+ Scrapy PREVIOUS_VERSION and earlier versions.++ Even though this is the default value for backward compatibility reasons,+ it is a deprecated value.++- ``'VERSION'``++ This implementation was introduced in Scrapy VERSION to fix an issue of the+ previous implementation.++ New projects should use this value. The :command:`startproject` command+ sets this value in the generated ``settings.py`` file.++If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are+using Scrapy components where changing the request fingerprinting algorithm+would cause undesired results, you need to carefully decide when to change the+value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`+setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request+fingerprinting algorithm and does not log this warning (+:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a+class).++Scenarios where changing the request fingerprinting algorithm may cause+undesired results include, for example, using the HTTP cache middleware (see+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).+Changing the request fingerprinting algorithm would invalidade the current+cache, requiring you to redownload all requests again.++Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in+your settings to switch already to the request fingerprinting implementation+that will be the only request fingerprinting implementation available in a+future version of Scrapy, and remove the deprecation warning triggered by using+the default value (``'PREVIOUS_VERSION'``).+++.. _PREVIOUS_VERSION-request-fingerprinter:+.. _custom-request-fingerprinter:++Writing your own request fingerprinter+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++A request fingerprinter is a class that must implement the following method:++.. method:: fingerprint(self, request)++ Return a :class:`bytes` object that uniquely identifies *request*.++ See also :ref:`request-fingerprint-restrictions`.++ :param request: request to fingerprint+ :type request: scrapy.http.Request++Additionally, it may also implement the following methods:++.. classmethod:: from_crawler(cls, crawler)++ If present, this class method is called to create a request fingerprinter+ instance from a :class:`~scrapy.crawler.Crawler` object. It must return a+ new instance of the request fingerprinter.++ *crawler* provides access to all Scrapy core components like settings and+ signals; it is a way for the request fingerprinter to access them and hook+ its functionality into Scrapy.++ :param crawler: crawler that uses this request fingerprinter+ :type crawler: :class:`~scrapy.crawler.Crawler` object++.. classmethod:: from_settings(cls, settings)++ If present, and ``from_crawler`` is not defined, this class method is called+ to create a request fingerprinter instance from a+ :class:`~scrapy.settings.Settings` object. It must return a new instance of+ the request fingerprinter.++The ``fingerprint`` method of the default request fingerprinter,+:class:`scrapy.utils.request.RequestFingerprinter`, uses+:func:`scrapy.utils.request.fingerprint` with its default parameters. For some+common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well+in your ``fingerprint`` method implementation:++.. autofunction:: scrapy.utils.request.fingerprint++For example, to take the value of a request header named ``X-ID`` into+account::++ # my_project/settings.py+ REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'++ # my_project/utils.py+ from scrapy.utils.request import fingerprint++ class RequestFingerprinter:++ def fingerprint(self, request):+ return fingerprint(request, include_headers=['X-ID'])++You can also write your own fingerprinting logic from scratch.++However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure+you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:++- Caching saves CPU by ensuring that fingerprints are calculated only once+ per request, and not once per Scrapy component that needs the fingerprint+ of a request.++- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that+ request objects do not stay in memory forever just because you have+ references to them in your cache dictionary.++For example, to take into account only the URL of a request, without any prior+URL canonicalization or taking the request method or body into account::++ from hashlib import sha1+ from weakref import WeakKeyDictionary++ from scrapy.utils.python import to_bytes++ class RequestFingerprinter:++ cache = WeakKeyDictionary()++ def fingerprint(self, request):+ if request not in self.cache:+ fp = sha1()+ fp.update(to_bytes(request.url))+ self.cache[request] = fp.digest()+ return self.cache[request]++If you need to be able to override the request fingerprinting for arbitrary+requests from your spider callbacks, you may implement a request fingerprinter+that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`+when available, and then falls back to+:func:`~scrapy.utils.request.fingerprint`. For example::++ from scrapy.utils.request import fingerprint++ class RequestFingerprinter:++ def fingerprint(self, request):+ if 'fingerprint' in request.meta:+ return request.meta['fingerprint']+ return fingerprint(request)++If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION+without using the deprecated ``'PREVIOUS_VERSION'`` value of the+:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following+request fingerprinter::++ from hashlib import sha1+ from weakref import WeakKeyDictionary++ from scrapy.utils.python import to_bytes+ from w3lib.url import canonicalize_url++ class RequestFingerprinter:++ cache = WeakKeyDictionary()++ def fingerprint(self, request):+ if request not in self.cache:+ fp = sha1()+ fp.update(to_bytes(request.method))+ fp.update(to_bytes(canonicalize_url(request.url)))+ fp.update(request.body or b'')+ self.cache[request] = fp.digest()+ return self.cache[request]+++.. _request-fingerprint-restrictions:++Request fingerprint restrictions+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++Scrapy components that use request fingerprints may impose additional+restrictions on the format of the fingerprints that your :ref:`request+fingerprinter <custom-request-fingerprinter>` generates.++The following built-in Scrapy components have such restrictions:++- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default+ value of :setting:`HTTPCACHE_STORAGE`)++ Request fingerprints must be at least 1 byte long.++ Path and filename length limits of the file system of+ :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,+ the following directory structure is created:++ - :attr:`Spider.name <scrapy.spiders.Spider.name>`++ - first byte of a request fingerprint as hexadecimal++ - fingerprint as hexadecimal++ - filenames up to 16 characters long++ For example, if a request fingerprint is made of 20 bytes (default),+ :setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,+ and the name of your spider is ``'my_spider'`` your file system must+ support a file path like::++ /home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers++- :class:`scrapy.extensions.httpcache.DbmCacheStorage`++ The underlying DBM implementation must support keys as long as twice+ the number of bytes of a request fingerprint, plus 5. For example,+ if a request fingerprint is made of 20 bytes (default),+ 45-character-long keys must be supported.++.. _topics-request-meta:Request.meta special keysdiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 4e105642d..2046c6446 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -825,12 +825,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``The class used to detect and filter duplicate requests.-The default (``RFPDupeFilter``) filters based on request fingerprint using-the ``scrapy.utils.request.request_fingerprint`` function. In order to change-the way duplicates are checked you could subclass ``RFPDupeFilter`` and-override its ``request_fingerprint`` method. This method should accept-scrapy :class:`~scrapy.Request` object and return its fingerprint-(a string).+The default (``RFPDupeFilter``) filters based on the+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.You can disable filtering of duplicate requests by setting:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex a638254f1..fdca7b335 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -51,6 +51,7 @@ class Crawler:self.spidercls.update_settings(self.settings)self.signals = SignalManager(self)+self.stats = load_object(self.settings['STATS_CLASS'])(self)handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))@@ -71,6 +72,12 @@ class Crawler:lf_cls = load_object(self.settings['LOG_FORMATTER'])self.logformatter = lf_cls.from_crawler(self)+ self.request_fingerprinter = create_instance(+ load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),+ settings=self.settings,+ crawler=self,+ )+reactor_class = self.settings.get("TWISTED_REACTOR")if init_reactor:# this needs to be done after the spider settings are merged,diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex 292c68099..d1b0559ef 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -1,14 +1,16 @@import loggingimport osfrom typing import Optional, Set, Type, TypeVar+from warnings import warnfrom twisted.internet.defer import Deferredfrom scrapy.http.request import Requestfrom scrapy.settings import BaseSettingsfrom scrapy.spiders import Spider+from scrapy.utils.deprecate import ScrapyDeprecationWarningfrom scrapy.utils.job import job_dir-from scrapy.utils.request import referer_str, request_fingerprint+from scrapy.utils.request import referer_str, RequestFingerprinterBaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")@@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")class RFPDupeFilter(BaseDupeFilter):"""Request Fingerprint duplicates filter"""- def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:+ def __init__(+ self,+ path: Optional[str] = None,+ debug: bool = False,+ *,+ fingerprinter=None,+ ) -> None:self.file = None+ self.fingerprinter = fingerprinter or RequestFingerprinter()self.fingerprints: Set[str] = set()self.logdupes = Trueself.debug = debug@@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):self.fingerprints.update(x.rstrip() for x in self.file)@classmethod- def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:+ def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:debug = settings.getbool('DUPEFILTER_DEBUG')- return cls(job_dir(settings), debug)+ try:+ return cls(job_dir(settings), debug, fingerprinter=fingerprinter)+ except TypeError:+ warn(+ "RFPDupeFilter subclasses must either modify their '__init__' "+ "method to support a 'fingerprinter' parameter or reimplement "+ "the 'from_settings' class method.",+ ScrapyDeprecationWarning,+ )+ result = cls(job_dir(settings), debug)+ result.fingerprinter = fingerprinter+ return result++ @classmethod+ def from_crawler(cls, crawler):+ try:+ return cls.from_settings(+ crawler.settings,+ fingerprinter=crawler.request_fingerprinter,+ )+ except TypeError:+ warn(+ "RFPDupeFilter subclasses must either modify their overridden "+ "'__init__' method and 'from_settings' class method to "+ "support a 'fingerprinter' parameter, or reimplement the "+ "'from_crawler' class method.",+ ScrapyDeprecationWarning,+ )+ result = cls.from_settings(crawler.settings)+ result.fingerprinter = crawler.request_fingerprinter+ return resultdef request_seen(self, request: Request) -> bool:fp = self.request_fingerprint(request)@@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):return Falsedef request_fingerprint(self, request: Request) -> str:- return request_fingerprint(request)+ return self.fingerprinter.fingerprint(request).hex()def close(self, reason: str) -> None:if self.file:diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex d0ae29b90..c71484cfa 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypesfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.project import data_pathfrom scrapy.utils.python import to_bytes, to_unicode-from scrapy.utils.request import request_fingerprintlogger = logging.getLogger(__name__)@@ -228,6 +227,8 @@ class DbmCacheStorage:logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})+ self._fingerprinter = spider.crawler.request_fingerprinter+def close_spider(self, spider):self.db.close()@@ -244,7 +245,7 @@ class DbmCacheStorage:return responsedef store_response(self, spider, request, response):- key = self._request_key(request)+ key = self._fingerprinter.fingerprint(request).hex()data = {'status': response.status,'url': response.url,@@ -255,7 +256,7 @@ class DbmCacheStorage:self.db[f'{key}_time'] = str(time())def _read_data(self, spider, request):- key = self._request_key(request)+ key = self._fingerprinter.fingerprint(request).hex()db = self.dbtkey = f'{key}_time'if tkey not in db:@@ -267,9 +268,6 @@ class DbmCacheStorage:return pickle.loads(db[f'{key}_data'])- def _request_key(self, request):- return request_fingerprint(request)-class FilesystemCacheStorage:@@ -283,6 +281,8 @@ class FilesystemCacheStorage:logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},extra={'spider': spider})+ self._fingerprinter = spider.crawler.request_fingerprinter+def close_spider(self, spider):pass@@ -329,7 +329,7 @@ class FilesystemCacheStorage:f.write(request.body)def _get_request_path(self, spider, request):- key = request_fingerprint(request)+ key = self._fingerprinter.fingerprint(request).hex()return os.path.join(self.cachedir, spider.name, key[0:2], key)def _read_meta(self, spider, request):diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 430c37227..5308a9793 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -11,7 +11,6 @@ from scrapy.settings import Settingsfrom scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.defer import mustbe_deferred, defer_resultfrom scrapy.utils.deprecate import ScrapyDeprecationWarning-from scrapy.utils.request import request_fingerprintfrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.log import failure_to_exc_info@@ -77,6 +76,7 @@ class MediaPipeline:except AttributeError:pipe = cls()pipe.crawler = crawler+ pipe._fingerprinter = crawler.request_fingerprinterreturn pipedef open_spider(self, spider):@@ -90,7 +90,7 @@ class MediaPipeline:return dfd.addCallback(self.item_completed, item, info)def _process_request(self, request, info, item):- fp = request_fingerprint(request)+ fp = self._fingerprinter.fingerprint(request)cb = request.callback or (lambda _: _)eb = request.errbackrequest.callback = Nonediff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 8389a70cb..f5a3efe69 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -246,6 +246,9 @@ REDIRECT_PRIORITY_ADJUST = +2REFERER_ENABLED = TrueREFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'+REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'+REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'+RETRY_ENABLED = TrueRETRY_TIMES = 2 # initial response + 2 retries = 3 requestsRETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmplindex a414b5fde..5e541e2c0 100644--- a/scrapy/templates/project/module/settings.py.tmpl+++ b/scrapy/templates/project/module/settings.py.tmpl@@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True#HTTPCACHE_DIR = 'httpcache'#HTTPCACHE_IGNORE_HTTP_CODES = []#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'++# Set settings whose default value is deprecated to a future-proof value+REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 70ef3ba2b..cf33317ce 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -4,7 +4,9 @@ scrapy.http.Request objects"""import hashlib-from typing import Dict, Iterable, Optional, Tuple, Union+import json+import warnings+from typing import Dict, Iterable, List, Optional, Tuple, Unionfrom urllib.parse import urlunparsefrom weakref import WeakKeyDictionary@@ -12,13 +14,22 @@ from w3lib.http import basic_auth_headerfrom w3lib.url import canonicalize_urlfrom scrapy import Request, Spider+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import load_objectfrom scrapy.utils.python import to_bytes, to_unicode-_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"-_fingerprint_cache = WeakKeyDictionary()+_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"+_deprecated_fingerprint_cache = WeakKeyDictionary()+++def _serialize_headers(headers, request):+ for header in headers:+ if header in request.headers:+ yield header+ for value in request.headers.getlist(header):+ yield valuedef request_fingerprint(@@ -27,7 +38,7 @@ def request_fingerprint(keep_fragments: bool = False,) -> str:"""- Return the request fingerprint.+ Return the request fingerprint as an hexadecimal string.The request fingerprint is a hash that uniquely identifies the resource therequest points to. For example, take the following two urls:@@ -43,7 +54,7 @@ def request_fingerprint(http://www.example.com/members/offers.html- Lot of sites use a cookie to store the session id, which adds a random+ Lots of sites use a cookie to store the session id, which adds a randomcomponent to the HTTP Request and thus should be ignored when calculatingthe fingerprint.@@ -55,29 +66,213 @@ def request_fingerprint(so they are also ignored by default when calculating the fingerprint.If you want to include them, set the keep_fragments argument to True(for instance when handling requests with a headless browser).-"""- headers: Optional[Tuple[bytes, ...]] = None+ if include_headers or keep_fragments:+ message = (+ 'Call to deprecated function '+ 'scrapy.utils.request.request_fingerprint().\n'+ '\n'+ 'If you are using this function in a Scrapy component because you '+ 'need a non-default fingerprinting algorithm, and you are OK '+ 'with that non-default fingerprinting algorithm being used by '+ 'all Scrapy components and not just the one calling this '+ 'function, use crawler.request_fingerprinter.fingerprint() '+ 'instead in your Scrapy component (you can get the crawler '+ 'object from the \'from_crawler\' class method), and use the '+ '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your '+ 'non-default fingerprinting algorithm.\n'+ '\n'+ 'Otherwise, consider using the '+ 'scrapy.utils.request.fingerprint() function instead.\n'+ '\n'+ 'If you switch to \'fingerprint()\', or assign the '+ '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses '+ '\'fingerprint()\', the generated fingerprints will not only be '+ 'bytes instead of a string, but they will also be different from '+ 'those generated by \'request_fingerprint()\'. Before you switch, '+ 'make sure that you understand the consequences of this (e.g. '+ 'cache invalidation) and are OK with them; otherwise, consider '+ 'implementing your own function which returns the same '+ 'fingerprints as the deprecated \'request_fingerprint()\' function.'+ )+ else:+ message = (+ 'Call to deprecated function '+ 'scrapy.utils.request.request_fingerprint().\n'+ '\n'+ 'If you are using this function in a Scrapy component, and you '+ 'are OK with users of your component changing the fingerprinting '+ 'algorithm through settings, use '+ 'crawler.request_fingerprinter.fingerprint() instead in your '+ 'Scrapy component (you can get the crawler object from the '+ '\'from_crawler\' class method).\n'+ '\n'+ 'Otherwise, consider using the '+ 'scrapy.utils.request.fingerprint() function instead.\n'+ '\n'+ 'Either way, the resulting fingerprints will be returned as '+ 'bytes, not as a string, and they will also be different from '+ 'those generated by \'request_fingerprint()\'. Before you switch, '+ 'make sure that you understand the consequences of this (e.g. '+ 'cache invalidation) and are OK with them; otherwise, consider '+ 'implementing your own function which returns the same '+ 'fingerprints as the deprecated \'request_fingerprint()\' function.'+ )+ warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)+ processed_include_headers: Optional[Tuple[bytes, ...]] = Noneif include_headers:- headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))- cache = _fingerprint_cache.setdefault(request, {})- cache_key = (headers, keep_fragments)+ processed_include_headers = tuple(+ to_bytes(h.lower()) for h in sorted(include_headers)+ )+ cache = _deprecated_fingerprint_cache.setdefault(request, {})+ cache_key = (processed_include_headers, keep_fragments)if cache_key not in cache:fp = hashlib.sha1()fp.update(to_bytes(request.method))fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))fp.update(request.body or b'')- if headers:- for hdr in headers:- if hdr in request.headers:- fp.update(hdr)- for v in request.headers.getlist(hdr):- fp.update(v)+ if processed_include_headers:+ for part in _serialize_headers(processed_include_headers, request):+ fp.update(part)cache[cache_key] = fp.hexdigest()return cache[cache_key]-def request_authenticate(request: Request, username: str, password: str) -> None:+def _request_fingerprint_as_bytes(*args, **kwargs):+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ return bytes.fromhex(request_fingerprint(*args, **kwargs))+++_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"+_fingerprint_cache = WeakKeyDictionary()+++def fingerprint(+ request: Request,+ *,+ include_headers: Optional[Iterable[Union[bytes, str]]] = None,+ keep_fragments: bool = False,+) -> bytes:+ """+ Return the request fingerprint.++ The request fingerprint is a hash that uniquely identifies the resource the+ request points to. For example, take the following two urls:++ http://www.example.com/query?id=111&cat=222+ http://www.example.com/query?cat=222&id=111++ Even though those are two different URLs both point to the same resource+ and are equivalent (i.e. they should return the same response).++ Another example are cookies used to store session ids. Suppose the+ following page is only accessible to authenticated users:++ http://www.example.com/members/offers.html++ Lots of sites use a cookie to store the session id, which adds a random+ component to the HTTP Request and thus should be ignored when calculating+ the fingerprint.++ For this reason, request headers are ignored by default when calculating+ the fingerprint. If you want to include specific headers use the+ include_headers argument, which is a list of Request headers to include.++ Also, servers usually ignore fragments in urls when handling requests,+ so they are also ignored by default when calculating the fingerprint.+ If you want to include them, set the keep_fragments argument to True+ (for instance when handling requests with a headless browser).+ """+ processed_include_headers: Optional[Tuple[bytes, ...]] = None+ if include_headers:+ processed_include_headers = tuple(+ to_bytes(h.lower()) for h in sorted(include_headers)+ )+ cache = _fingerprint_cache.setdefault(request, {})+ cache_key = (processed_include_headers, keep_fragments)+ if cache_key not in cache:+ # To decode bytes reliably (JSON does not support bytes), regardless of+ # character encoding, we use bytes.hex()+ headers: Dict[str, List[str]] = {}+ if processed_include_headers:+ for header in processed_include_headers:+ if header in request.headers:+ headers[header.hex()] = [+ header_value.hex()+ for header_value in request.headers.getlist(header)+ ]+ fingerprint_data = {+ 'method': to_unicode(request.method),+ 'url': canonicalize_url(request.url, keep_fragments=keep_fragments),+ 'body': (request.body or b'').hex(),+ 'headers': headers,+ }+ fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)+ cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()+ return cache[cache_key]+++class RequestFingerprinter:+ """Default fingerprinter.++ It takes into account a canonical version+ (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url+ <scrapy.http.Request.url>` and the values of :attr:`request.method+ <scrapy.http.Request.method>` and :attr:`request.body+ <scrapy.http.Request.body>`. It then generates an `SHA1+ <https://en.wikipedia.org/wiki/SHA-1>`_ hash.++ .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`.+ """++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)++ def __init__(self, crawler=None):+ if crawler:+ implementation = crawler.settings.get(+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION'+ )+ else:+ implementation = 'PREVIOUS_VERSION'+ if implementation == 'PREVIOUS_VERSION':+ message = (+ '\'PREVIOUS_VERSION\' is a deprecated value for the '+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'+ '\n'+ 'It is also the default value. In other words, it is normal '+ 'to get this warning if you have not defined a value for the '+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so '+ 'for backward compatibility reasons, but it will change in a '+ 'future version of Scrapy.\n'+ '\n'+ 'See the documentation of the '+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for '+ 'information on how to handle this deprecation.'+ )+ warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)+ self._fingerprint = _request_fingerprint_as_bytes+ elif implementation == 'VERSION':+ self._fingerprint = fingerprint+ else:+ raise ValueError(+ f'Got an invalid value on setting '+ f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '+ f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '+ f'and \'VERSION\'.'+ )++ def fingerprint(self, request):+ return self._fingerprint(request)+++def request_authenticate(+ request: Request,+ username: str,+ password: str,+) -> None:"""Authenticate the given request (in place) using the HTTP basic accessauthentication mechanism (RFC 2617) and the given username and password"""diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 24c38283a..b90ea5009 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -54,7 +54,7 @@ def get_ftp_content_and_delete(return "".join(ftp_data)-def get_crawler(spidercls=None, settings_dict=None):+def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):"""Return an unconfigured Crawler object. If settings_dict is given, itwill be used to populate the crawler settings with a project levelpriority.@@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None):from scrapy.crawler import CrawlerRunnerfrom scrapy.spiders import Spider- runner = CrawlerRunner(settings_dict)+ # Set by default settings that prevent deprecation warnings.+ settings = {}+ if prevent_warnings:+ settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION'+ settings.update(settings_dict or {})+ runner = CrawlerRunner(settings)return runner.create_crawler(spidercls or Spider)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 8f6227109..f7aa769e4 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -104,7 +104,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):custom_settings = {'LOG_LEVEL': 'INFO','LOG_FILE': log_file,- # disable telnet if not available to avoid an extra warning+ # settings to avoid extra warnings+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION','TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,}diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 680bb6dc8..b7df2554a 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -15,6 +15,16 @@ from scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpider+def _get_dupefilter(*, crawler=None, settings=None, open=True):+ if crawler is None:+ crawler = get_crawler(settings_dict=settings)+ scheduler = Scheduler.from_crawler(crawler)+ dupefilter = scheduler.df+ if open:+ dupefilter.open()+ return dupefilter++class FromCrawlerRFPDupeFilter(RFPDupeFilter):@classmethod@@ -64,9 +74,7 @@ class RFPDupeFilterTest(unittest.TestCase):self.assertEqual(scheduler.df.method, 'n/a')def test_filter(self):- dupefilter = RFPDupeFilter()- dupefilter.open()-+ dupefilter = _get_dupefilter()r1 = Request('http://scrapytest.org/1')r2 = Request('http://scrapytest.org/2')r3 = Request('http://scrapytest.org/2')@@ -85,7 +93,7 @@ class RFPDupeFilterTest(unittest.TestCase):path = tempfile.mkdtemp()try:- df = RFPDupeFilter(path)+ df = _get_dupefilter(settings={'JOBDIR': path}, open=False)try:df.open()assert not df.request_seen(r1)@@ -93,7 +101,8 @@ class RFPDupeFilterTest(unittest.TestCase):finally:df.close('finished')- df2 = RFPDupeFilter(path)+ df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False)+ assert df != df2try:df2.open()assert df2.request_seen(r1)@@ -109,26 +118,24 @@ class RFPDupeFilterTest(unittest.TestCase):output of request_seen."""+ dupefilter = _get_dupefilter()r1 = Request('http://scrapytest.org/index.html')r2 = Request('http://scrapytest.org/INDEX.html')- dupefilter = RFPDupeFilter()- dupefilter.open()-assert not dupefilter.request_seen(r1)assert not dupefilter.request_seen(r2)dupefilter.close('finished')- class CaseInsensitiveRFPDupeFilter(RFPDupeFilter):+ class RequestFingerprinter:- def request_fingerprint(self, request):+ def fingerprint(self, request):fp = hashlib.sha1()fp.update(to_bytes(request.url.lower()))- return fp.hexdigest()+ return fp.digest()- case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter()- case_insensitive_dupefilter.open()+ settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter}+ case_insensitive_dupefilter = _get_dupefilter(settings=settings)assert not case_insensitive_dupefilter.request_seen(r1)assert case_insensitive_dupefilter.request_seen(r2)@@ -142,8 +149,10 @@ class RFPDupeFilterTest(unittest.TestCase):r1 = Request('http://scrapytest.org/1')path = tempfile.mkdtemp()+ crawler = get_crawler(settings_dict={'JOBDIR': path})try:- df = RFPDupeFilter(path)+ scheduler = Scheduler.from_crawler(crawler)+ df = scheduler.dfdf.open()df.request_seen(r1)df.close('finished')@@ -164,11 +173,8 @@ class RFPDupeFilterTest(unittest.TestCase):settings = {'DUPEFILTER_DEBUG': False,'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}crawler = get_crawler(SimpleSpider, settings_dict=settings)- scheduler = Scheduler.from_crawler(crawler)spider = SimpleSpider.from_crawler(crawler)-- dupefilter = scheduler.df- dupefilter.open()+ dupefilter = _get_dupefilter(crawler=crawler)r1 = Request('http://scrapytest.org/index.html')r2 = Request('http://scrapytest.org/index.html')@@ -193,11 +199,41 @@ class RFPDupeFilterTest(unittest.TestCase):settings = {'DUPEFILTER_DEBUG': True,'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}crawler = get_crawler(SimpleSpider, settings_dict=settings)- scheduler = Scheduler.from_crawler(crawler)spider = SimpleSpider.from_crawler(crawler)+ dupefilter = _get_dupefilter(crawler=crawler)++ r1 = Request('http://scrapytest.org/index.html')+ r2 = Request('http://scrapytest.org/index.html',+ headers={'Referer': 'http://scrapytest.org/INDEX.html'})- dupefilter = scheduler.df- dupefilter.open()+ dupefilter.log(r1, spider)+ dupefilter.log(r2, spider)++ assert crawler.stats.get_value('dupefilter/filtered') == 2+ log.check_present(+ (+ 'scrapy.dupefilters',+ 'DEBUG',+ 'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'+ )+ )+ log.check_present(+ (+ 'scrapy.dupefilters',+ 'DEBUG',+ 'Filtered duplicate request: <GET http://scrapytest.org/index.html>'+ ' (referer: http://scrapytest.org/INDEX.html)'+ )+ )++ dupefilter.close('finished')++ def test_log_debug_default_dupefilter(self):+ with LogCapture() as log:+ settings = {'DUPEFILTER_DEBUG': True}+ crawler = get_crawler(SimpleSpider, settings_dict=settings)+ spider = SimpleSpider.from_crawler(crawler)+ dupefilter = _get_dupefilter(crawler=crawler)r1 = Request('http://scrapytest.org/index.html')r2 = Request('http://scrapytest.org/index.html',diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 0ff2045ed..4228173ed 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -25,6 +25,7 @@ from scrapy.pipelines.files import (from scrapy.settings import Settingsfrom scrapy.utils.test import (assert_gcs_environ,+ get_crawler,get_ftp_content_and_delete,get_gcs_content_and_delete,skip_if_no_boto,@@ -47,7 +48,9 @@ class FilesPipelineTestCase(unittest.TestCase):def setUp(self):self.tempdir = mkdtemp()- self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir}))+ settings_dict = {'FILES_STORE': self.tempdir}+ crawler = get_crawler(spidercls=None, settings_dict=settings_dict)+ self.pipeline = FilesPipeline.from_crawler(crawler)self.pipeline.download_func = _mocked_download_funcself.pipeline.open_spider(None)diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex a802c7cf1..84e867660 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -7,17 +7,17 @@ from twisted.python.failure import Failurefrom twisted.internet import reactorfrom twisted.internet.defer import Deferred, inlineCallbacks+from scrapy import signalsfrom scrapy.http import Request, Responsefrom scrapy.settings import Settingsfrom scrapy.spiders import Spider-from scrapy.utils.deprecate import ScrapyDeprecationWarning-from scrapy.utils.request import request_fingerprint+from scrapy.pipelines.files import FileExceptionfrom scrapy.pipelines.images import ImagesPipelinefrom scrapy.pipelines.media import MediaPipeline-from scrapy.pipelines.files import FileException+from scrapy.utils.deprecate import ScrapyDeprecationWarningfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.signal import disconnect_all-from scrapy import signals+from scrapy.utils.test import get_crawlertry:@@ -39,11 +39,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase):settings = Nonedef setUp(self):- self.spider = Spider('media.com')- self.pipe = self.pipeline_class(download_func=_mocked_download_func,- settings=Settings(self.settings))+ spider_cls = Spider+ self.spider = spider_cls('media.com')+ crawler = get_crawler(spider_cls, self.settings)+ self.pipe = self.pipeline_class.from_crawler(crawler)+ self.pipe.download_func = _mocked_download_funcself.pipe.open_spider(self.spider)self.info = self.pipe.spiderinfo+ self.fingerprint = crawler.request_fingerprinter.fingerprintdef tearDown(self):for name, signal in vars(signals).items():@@ -156,7 +159,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):self.assertEqual(failure.value.__context__, def_gen_return_exc)# Let's calculate the request fingerprint and fake some runtime data...- fp = request_fingerprint(request)+ fp = self.fingerprint(request)info = self.pipe.spiderinfoinfo.downloading.add(fp)info.waiting[fp] = []@@ -273,7 +276,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):item = dict(requests=req) # pass a single itemnew_item = yield self.pipe.process_item(item, self.spider)assert new_item is item- assert request_fingerprint(req) in self.info.downloaded+ self.assertIn(self.fingerprint(req), self.info.downloaded)# returns iterable of Requestsreq1 = Request('http://url1')@@ -281,8 +284,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):item = dict(requests=iter([req1, req2]))new_item = yield self.pipe.process_item(item, self.spider)assert new_item is item- assert request_fingerprint(req1) in self.info.downloaded- assert request_fingerprint(req2) in self.info.downloaded+ assert self.fingerprint(req1) in self.info.downloaded+ assert self.fingerprint(req2) in self.info.downloaded@inlineCallbacksdef test_results_are_cached_across_multiple_items(self):@@ -298,7 +301,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):item = dict(requests=req2)new_item = yield self.pipe.process_item(item, self.spider)self.assertTrue(new_item is item)- self.assertEqual(request_fingerprint(req1), request_fingerprint(req2))+ self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))self.assertEqual(new_item['results'], [(True, rsp1)])@inlineCallbacks@@ -314,7 +317,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):@inlineCallbacksdef test_wait_if_request_is_downloading(self):def _check_downloading(response):- fp = request_fingerprint(req1)+ fp = self.fingerprint(req1)self.assertTrue(fp in self.info.downloading)self.assertTrue(fp in self.info.waiting)self.assertTrue(fp not in self.info.downloaded)@@ -351,7 +354,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):def __init__(self, *args, **kwargs):- super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs)+ super().__init__(*args, **kwargs)self._mockcalled = []def get_media_requests(self, item, info):@@ -369,19 +372,19 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):def media_to_download(self, request, info):self._mockcalled.append('media_to_download')- return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info)+ return super().media_to_download(request, info)def media_downloaded(self, response, request, info):self._mockcalled.append('media_downloaded')- return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info)+ return super().media_downloaded(response, request, info)def file_downloaded(self, response, request, info):self._mockcalled.append('file_downloaded')- return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info)+ return super().file_downloaded(response, request, info)def file_path(self, request, response=None, info=None):self._mockcalled.append('file_path')- return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)+ return super().file_path(request, response, info)def thumb_path(self, request, thumb_id, response=None, info=None):self._mockcalled.append('thumb_path')@@ -393,18 +396,20 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):def image_downloaded(self, response, request, info):self._mockcalled.append('image_downloaded')- return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info)+ return super().image_downloaded(response, request, info)class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):skip = skip_pillowdef setUp(self):- self.pipe = MockedMediaPipelineDeprecatedMethods(- store_uri='store-uri',- download_func=_mocked_download_func,- settings=Settings({"IMAGES_THUMBS": {'small': (50, 50)}})- )+ settings_dict = {+ 'IMAGES_STORE': 'store-uri',+ 'IMAGES_THUMBS': {'small': (50, 50)},+ }+ crawler = get_crawler(spidercls=None, settings_dict=settings_dict)+ self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)+ self.pipe.download_func = _mocked_download_funcself.pipe.open_spider(None)self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 7e0049b1d..e9edfee98 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -1,33 +1,166 @@import unittest+import warnings+from hashlib import sha1+from typing import Dict, Mapping, Optional, Tuple, Union+from weakref import WeakKeyDictionary++import pytest+from w3lib.url import canonicalize_url+from scrapy.http import Request+from scrapy.utils.deprecate import ScrapyDeprecationWarning+from scrapy.utils.python import to_bytesfrom scrapy.utils.request import (+ _deprecated_fingerprint_cache,_fingerprint_cache,+ _request_fingerprint_as_bytes,+ fingerprint,request_authenticate,request_fingerprint,request_httprepr,)+from scrapy.utils.test import get_crawlerclass UtilsRequestTest(unittest.TestCase):- def test_request_fingerprint(self):+ def test_request_authenticate(self):+ r = Request("http://www.example.com")+ request_authenticate(r, 'someuser', 'somepass')+ self.assertEqual(r.headers['Authorization'], b'Basic c29tZXVzZXI6c29tZXBhc3M=')++ def test_request_httprepr(self):+ r1 = Request("http://www.example.com")+ self.assertEqual(request_httprepr(r1), b'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n')++ r1 = Request("http://www.example.com/some/page.html?arg=1")+ self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')++ r1 = Request("http://www.example.com", method='POST',+ headers={"Content-type": b"text/html"}, body=b"Some body")+ self.assertEqual(+ request_httprepr(r1),+ b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body'+ )++ def test_request_httprepr_for_non_http_request(self):+ # the representation is not important but it must not fail.+ request_httprepr(Request("file:///tmp/foo.txt"))+ request_httprepr(Request("ftp://localhost/tmp/foo.txt"))+++class FingerprintTest(unittest.TestCase):+ maxDiff = None++ function = staticmethod(fingerprint)+ cache: Union[+ "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]",+ "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]",+ ] = _fingerprint_cache+ default_cache_key = (None, False)+ known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (+ (+ Request("http://example.org"),+ b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD',+ {},+ ),+ (+ Request("https://example.org"),+ b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',+ {},+ ),+ (+ Request("https://example.org?a"),+ b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c',+ {},+ ),+ (+ Request("https://example.org?a=b"),+ b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17',+ {},+ ),+ (+ Request("https://example.org?a=b&a"),+ b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96',+ {},+ ),+ (+ Request("https://example.org?a=b&a=c"),+ b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef',+ {},+ ),+ (+ Request("https://example.org", method='POST'),+ b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d',+ {},+ ),+ (+ Request("https://example.org", body=b'a'),+ b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_',+ {},+ ),+ (+ Request("https://example.org", method='POST', body=b'a'),+ b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7',+ {},+ ),+ (+ Request("https://example.org#a", headers={'A': b'B'}),+ b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',+ {},+ ),+ (+ Request("https://example.org#a", headers={'A': b'B'}),+ b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.',+ {'include_headers': ['A']},+ ),+ (+ Request("https://example.org#a", headers={'A': b'B'}),+ b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f',+ {'keep_fragments': True},+ ),+ (+ Request("https://example.org#a", headers={'A': b'B'}),+ b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11',+ {'include_headers': ['A'], 'keep_fragments': True},+ ),+ (+ Request("https://example.org/ab"),+ b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6',+ {},+ ),+ (+ Request("https://example.org/a", body=b'b'),+ b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5',+ {},+ ),+ )++ def test_query_string_key_order(self):r1 = Request("http://www.example.com/query?id=111&cat=222")r2 = Request("http://www.example.com/query?cat=222&id=111")- self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))+ self.assertEqual(self.function(r1), self.function(r1))+ self.assertEqual(self.function(r1), self.function(r2))+ def test_query_string_key_without_value(self):r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')- self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))+ self.assertNotEqual(self.function(r1), self.function(r2))- # make sure caching is working- self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)])+ def test_caching(self):+ r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')+ self.assertEqual(+ self.function(r1),+ self.cache[r1][self.default_cache_key]+ )+ def test_header(self):r1 = Request("http://www.example.com/members/offers.html")r2 = Request("http://www.example.com/members/offers.html")r2.headers['SESSIONID'] = b"somehash"- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))+ self.assertEqual(self.function(r1), self.function(r2))+ def test_headers(self):r1 = Request("http://www.example.com/")r2 = Request("http://www.example.com/")r2.headers['Accept-Language'] = b'en'@@ -35,62 +168,512 @@ class UtilsRequestTest(unittest.TestCase):r3.headers['Accept-Language'] = b'en'r3.headers['SESSIONID'] = b"somehash"- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3))+ self.assertEqual(self.function(r1), self.function(r2), self.function(r3))- self.assertEqual(request_fingerprint(r1),- request_fingerprint(r1, include_headers=['Accept-Language']))+ self.assertEqual(self.function(r1),+ self.function(r1, include_headers=['Accept-Language']))self.assertNotEqual(- request_fingerprint(r1),- request_fingerprint(r2, include_headers=['Accept-Language']))+ self.function(r1),+ self.function(r2, include_headers=['Accept-Language']))- self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),- request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))+ self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']),+ self.function(r3, include_headers=['SESSIONID', 'Accept-Language']))+ def test_fragment(self):r1 = Request("http://www.example.com/test.html")r2 = Request("http://www.example.com/test.html#fragment")- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))- self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True))- self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True))- self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True))+ self.assertEqual(self.function(r1), self.function(r2))+ self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True))+ self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True))+ self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True))+ def test_method_and_body(self):r1 = Request("http://www.example.com")r2 = Request("http://www.example.com", method='POST')r3 = Request("http://www.example.com", method='POST', body=b'request body')- self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))- self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3))+ self.assertNotEqual(self.function(r1), self.function(r2))+ self.assertNotEqual(self.function(r2), self.function(r3))+ def test_request_replace(self):# cached fingerprint must be cleared on request copyr1 = Request("http://www.example.com")- fp1 = request_fingerprint(r1)+ fp1 = self.function(r1)r2 = r1.replace(url="http://www.example.com/other")- fp2 = request_fingerprint(r2)+ fp2 = self.function(r2)self.assertNotEqual(fp1, fp2)- def test_request_authenticate(self):- r = Request("http://www.example.com")- request_authenticate(r, 'someuser', 'somepass')- self.assertEqual(r.headers['Authorization'], b'Basic c29tZXVzZXI6c29tZXBhc3M=')+ def test_part_separation(self):+ # An old implementation used to serialize request data in a way that+ # would put the body right after the URL.+ r1 = Request("http://www.example.com/foo")+ fp1 = self.function(r1)+ r2 = Request("http://www.example.com/f", body=b'oo')+ fp2 = self.function(r2)+ self.assertNotEqual(fp1, fp2)- def test_request_httprepr(self):- r1 = Request("http://www.example.com")- self.assertEqual(request_httprepr(r1), b'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n')+ def test_hashes(self):+ """Test hardcoded hashes, to make sure future changes to not introduce+ backward incompatibilities."""+ actual = [+ self.function(request, **kwargs)+ for request, _, kwargs in self.known_hashes+ ]+ expected = [+ _fingerprint+ for _, _fingerprint, _ in self.known_hashes+ ]+ self.assertEqual(actual, expected)- r1 = Request("http://www.example.com/some/page.html?arg=1")- self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')- r1 = Request("http://www.example.com", method='POST',- headers={"Content-type": b"text/html"}, body=b"Some body")+class RequestFingerprintTest(FingerprintTest):+ function = staticmethod(request_fingerprint)+ cache = _deprecated_fingerprint_cache+ known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (+ (+ Request("http://example.org"),+ 'b2e5245ef826fd9576c93bd6e392fce3133fab62',+ {},+ ),+ (+ Request("https://example.org"),+ 'bd10a0a89ea32cdee77917320f1309b0da87e892',+ {},+ ),+ (+ Request("https://example.org?a"),+ '2fb7d48ae02f04b749f40caa969c0bc3c43204ce',+ {},+ ),+ (+ Request("https://example.org?a=b"),+ '42e5fe149b147476e3f67ad0670c57b4cc57856a',+ {},+ ),+ (+ Request("https://example.org?a=b&a"),+ 'd23a9787cb56c6375c2cae4453c5a8c634526942',+ {},+ ),+ (+ Request("https://example.org?a=b&a=c"),+ '9a18a7a8552a9182b7f1e05d33876409e421e5c5',+ {},+ ),+ (+ Request("https://example.org", method='POST'),+ 'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6',+ {},+ ),+ (+ Request("https://example.org", body=b'a'),+ '4bb136e54e715a4ea7a9dd1101831765d33f2d60',+ {},+ ),+ (+ Request("https://example.org", method='POST', body=b'a'),+ '6c6595374a304b293be762f7b7be3f54e9947c65',+ {},+ ),+ (+ Request("https://example.org#a", headers={'A': b'B'}),+ 'bd10a0a89ea32cdee77917320f1309b0da87e892',+ {},+ ),+ (+ Request("https://example.org#a", headers={'A': b'B'}),+ '515b633cb3ca502a33a9d8c890e889ec1e425e65',+ {'include_headers': ['A']},+ ),+ (+ Request("https://example.org#a", headers={'A': b'B'}),+ '505c96e7da675920dfef58725e8c957dfdb38f47',+ {'keep_fragments': True},+ ),+ (+ Request("https://example.org#a", headers={'A': b'B'}),+ 'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da',+ {'include_headers': ['A'], 'keep_fragments': True},+ ),+ (+ Request("https://example.org/ab"),+ '4e2870fee58582d6f81755e9b8fdefe3cba0c951',+ {},+ ),+ (+ Request("https://example.org/a", body=b'b'),+ '4e2870fee58582d6f81755e9b8fdefe3cba0c951',+ {},+ ),+ )++ @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)+ def test_part_separation(self):+ super().test_part_separation()++ def test_deprecation_default_parameters(self):+ with pytest.warns(ScrapyDeprecationWarning) as warnings:+ self.function(Request("http://www.example.com"))+ messages = [str(warning.message) for warning in warnings]+ self.assertTrue(+ any(+ 'Call to deprecated function' in message+ for message in messages+ )+ )+ self.assertFalse(any('non-default' in message for message in messages))++ def test_deprecation_non_default_parameters(self):+ with pytest.warns(ScrapyDeprecationWarning) as warnings:+ self.function(Request("http://www.example.com"), keep_fragments=True)+ messages = [str(warning.message) for warning in warnings]+ self.assertTrue(+ any(+ 'Call to deprecated function' in message+ for message in messages+ )+ )+ self.assertTrue(any('non-default' in message for message in messages))+++class RequestFingerprintAsBytesTest(FingerprintTest):+ function = staticmethod(_request_fingerprint_as_bytes)+ cache = _deprecated_fingerprint_cache+ known_hashes = RequestFingerprintTest.known_hashes++ def test_caching(self):+ r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')self.assertEqual(- request_httprepr(r1),- b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body'+ self.function(r1),+ bytes.fromhex(self.cache[r1][self.default_cache_key]))- def test_request_httprepr_for_non_http_request(self):- # the representation is not important but it must not fail.- request_httprepr(Request("file:///tmp/foo.txt"))- request_httprepr(Request("ftp://localhost/tmp/foo.txt"))+ @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)+ def test_part_separation(self):+ super().test_part_separation()++ def test_hashes(self):+ actual = [+ self.function(request, **kwargs)+ for request, _, kwargs in self.known_hashes+ ]+ expected = [+ bytes.fromhex(_fingerprint)+ for _, _fingerprint, _ in self.known_hashes+ ]+ self.assertEqual(actual, expected)+++_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()+++def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):+ if include_headers:+ include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))+ cache = _fingerprint_cache_2_6.setdefault(request, {})+ cache_key = (include_headers, keep_fragments)+ if cache_key not in cache:+ fp = sha1()+ fp.update(to_bytes(request.method))+ fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))+ fp.update(request.body or b'')+ if include_headers:+ for hdr in include_headers:+ if hdr in request.headers:+ fp.update(hdr)+ for v in request.headers.getlist(hdr):+ fp.update(v)+ cache[cache_key] = fp.hexdigest()+ return cache[cache_key]+++REQUEST_OBJECTS_TO_TEST = (+ Request("http://www.example.com/"),+ Request("http://www.example.com/query?id=111&cat=222"),+ Request("http://www.example.com/query?cat=222&id=111"),+ Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'),+ Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'),+ Request("http://www.example.com/members/offers.html"),+ Request(+ "http://www.example.com/members/offers.html",+ headers={'SESSIONID': b"somehash"},+ ),+ Request(+ "http://www.example.com/",+ headers={'Accept-Language': b"en"},+ ),+ Request(+ "http://www.example.com/",+ headers={+ 'Accept-Language': b"en",+ 'SESSIONID': b"somehash",+ },+ ),+ Request("http://www.example.com/test.html"),+ Request("http://www.example.com/test.html#fragment"),+ Request("http://www.example.com", method='POST'),+ Request("http://www.example.com", method='POST', body=b'request body'),+)+++class BackwardCompatibilityTestCase(unittest.TestCase):++ def test_function_backward_compatibility(self):+ include_headers_to_test = (+ None,+ ['Accept-Language'],+ ['accept-language', 'sessionid'],+ ['SESSIONID', 'Accept-Language'],+ )+ for request_object in REQUEST_OBJECTS_TO_TEST:+ for include_headers in include_headers_to_test:+ for keep_fragments in (False, True):+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ fp = request_fingerprint(+ request_object,+ include_headers=include_headers,+ keep_fragments=keep_fragments,+ )+ old_fp = request_fingerprint_2_6(+ request_object,+ include_headers=include_headers,+ keep_fragments=keep_fragments,+ )+ self.assertEqual(fp, old_fp)++ def test_component_backward_compatibility(self):+ for request_object in REQUEST_OBJECTS_TO_TEST:+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ crawler = get_crawler(prevent_warnings=False)+ fp = crawler.request_fingerprinter.fingerprint(request_object)+ old_fp = request_fingerprint_2_6(request_object)+ self.assertEqual(fp.hex(), old_fp)++ def test_custom_component_backward_compatibility(self):+ """Tests that the backward-compatible request fingerprinting class featured+ in the documentation is indeed backward compatible and does not cause a+ warning to be logged."""++ class RequestFingerprinter:++ cache = WeakKeyDictionary()++ def fingerprint(self, request):+ if request not in self.cache:+ fp = sha1()+ fp.update(to_bytes(request.method))+ fp.update(to_bytes(canonicalize_url(request.url)))+ fp.update(request.body or b'')+ self.cache[request] = fp.digest()+ return self.cache[request]++ for request_object in REQUEST_OBJECTS_TO_TEST:+ with warnings.catch_warnings() as logged_warnings:+ settings = {+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,+ }+ crawler = get_crawler(settings_dict=settings)+ fp = crawler.request_fingerprinter.fingerprint(request_object)+ old_fp = request_fingerprint_2_6(request_object)+ self.assertEqual(fp.hex(), old_fp)+ self.assertFalse(logged_warnings)+++class RequestFingerprinterTestCase(unittest.TestCase):++ def test_default_implementation(self):+ with warnings.catch_warnings(record=True) as logged_warnings:+ crawler = get_crawler(prevent_warnings=False)+ request = Request('https://example.com')+ self.assertEqual(+ crawler.request_fingerprinter.fingerprint(request),+ _request_fingerprint_as_bytes(request),+ )+ self.assertTrue(logged_warnings)++ def test_deprecated_implementation(self):+ settings = {+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION',+ }+ with warnings.catch_warnings(record=True) as logged_warnings:+ crawler = get_crawler(settings_dict=settings)+ request = Request('https://example.com')+ self.assertEqual(+ crawler.request_fingerprinter.fingerprint(request),+ _request_fingerprint_as_bytes(request),+ )+ self.assertTrue(logged_warnings)++ def test_recommended_implementation(self):+ settings = {+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',+ }+ with warnings.catch_warnings(record=True) as logged_warnings:+ crawler = get_crawler(settings_dict=settings)+ request = Request('https://example.com')+ self.assertEqual(+ crawler.request_fingerprinter.fingerprint(request),+ fingerprint(request),+ )+ self.assertFalse(logged_warnings)++ def test_unknown_implementation(self):+ settings = {+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5',+ }+ with self.assertRaises(ValueError):+ get_crawler(settings_dict=settings)+++class CustomRequestFingerprinterTestCase(unittest.TestCase):++ def test_include_headers(self):++ class RequestFingerprinter:++ def fingerprint(self, request):+ return fingerprint(request, include_headers=['X-ID'])++ settings = {+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,+ }+ crawler = get_crawler(settings_dict=settings)++ r1 = Request("http://www.example.com", headers={'X-ID': '1'})+ fp1 = crawler.request_fingerprinter.fingerprint(r1)+ r2 = Request("http://www.example.com", headers={'X-ID': '2'})+ fp2 = crawler.request_fingerprinter.fingerprint(r2)+ self.assertNotEqual(fp1, fp2)++ def test_dont_canonicalize(self):++ class RequestFingerprinter:+ cache = WeakKeyDictionary()++ def fingerprint(self, request):+ if request not in self.cache:+ fp = sha1()+ fp.update(to_bytes(request.url))+ self.cache[request] = fp.digest()+ return self.cache[request]++ settings = {+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,+ }+ crawler = get_crawler(settings_dict=settings)++ r1 = Request("http://www.example.com?a=1&a=2")+ fp1 = crawler.request_fingerprinter.fingerprint(r1)+ r2 = Request("http://www.example.com?a=2&a=1")+ fp2 = crawler.request_fingerprinter.fingerprint(r2)+ self.assertNotEqual(fp1, fp2)++ def test_meta(self):++ class RequestFingerprinter:++ def fingerprint(self, request):+ if 'fingerprint' in request.meta:+ return request.meta['fingerprint']+ return fingerprint(request)++ settings = {+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,+ }+ crawler = get_crawler(settings_dict=settings)++ r1 = Request("http://www.example.com")+ fp1 = crawler.request_fingerprinter.fingerprint(r1)+ r2 = Request("http://www.example.com", meta={'fingerprint': 'a'})+ fp2 = crawler.request_fingerprinter.fingerprint(r2)+ r3 = Request("http://www.example.com", meta={'fingerprint': 'a'})+ fp3 = crawler.request_fingerprinter.fingerprint(r3)+ r4 = Request("http://www.example.com", meta={'fingerprint': 'b'})+ fp4 = crawler.request_fingerprinter.fingerprint(r4)+ self.assertNotEqual(fp1, fp2)+ self.assertNotEqual(fp1, fp4)+ self.assertNotEqual(fp2, fp4)+ self.assertEqual(fp2, fp3)++ def test_from_crawler(self):++ class RequestFingerprinter:++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)++ def __init__(self, crawler):+ self._fingerprint = crawler.settings['FINGERPRINT']++ def fingerprint(self, request):+ return self._fingerprint++ settings = {+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,+ 'FINGERPRINT': b'fingerprint',+ }+ crawler = get_crawler(settings_dict=settings)++ request = Request("http://www.example.com")+ fingerprint = crawler.request_fingerprinter.fingerprint(request)+ self.assertEqual(fingerprint, settings['FINGERPRINT'])++ def test_from_settings(self):++ class RequestFingerprinter:++ @classmethod+ def from_settings(cls, settings):+ return cls(settings)++ def __init__(self, settings):+ self._fingerprint = settings['FINGERPRINT']++ def fingerprint(self, request):+ return self._fingerprint++ settings = {+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,+ 'FINGERPRINT': b'fingerprint',+ }+ crawler = get_crawler(settings_dict=settings)++ request = Request("http://www.example.com")+ fingerprint = crawler.request_fingerprinter.fingerprint(request)+ self.assertEqual(fingerprint, settings['FINGERPRINT'])++ def test_from_crawler_and_settings(self):++ class RequestFingerprinter:++ # This method is ignored due to the presence of from_crawler+ @classmethod+ def from_settings(cls, settings):+ return cls(settings)++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)++ def __init__(self, crawler):+ self._fingerprint = crawler.settings['FINGERPRINT']++ def fingerprint(self, request):+ return self._fingerprint++ settings = {+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,+ 'FINGERPRINT': b'fingerprint',+ }+ crawler = get_crawler(settings_dict=settings)++ request = Request("http://www.example.com")+ fingerprint = crawler.request_fingerprinter.fingerprint(request)+ self.assertEqual(fingerprint, settings['FINGERPRINT'])if __name__ == "__main__":