Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Logging the request referer when DUPEFILTER_DEBUG is active

Júlio César Batistabdf12f775062
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 9d8966b9c..0bcdd3495 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -3,8 +3,7 @@ import os
import logging
from scrapy.utils.job import job_dir
-from scrapy.utils.request import request_fingerprint
-
+from scrapy.utils.request import referer_str, request_fingerprint
class BaseDupeFilter(object):
@@ -61,8 +60,9 @@ class RFPDupeFilter(BaseDupeFilter):
def log(self, request, spider):
if self.debug:
- msg = "Filtered duplicate request: %(request)s"
- self.logger.debug(msg, {'request': request}, extra={'spider': spider})
+ msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
+ args = {'request': request, 'referer': referer_str(request) }
+ self.logger.debug(msg, args, extra={'spider': spider})
elif self.logdupes:
msg = ("Filtered duplicate request: %(request)s"
" - no more duplicates will be shown"