Scrapy, a fast high-level web crawling & scraping framework for Python.
fix: return unique_list only when link_extractor.unique is True
Silvio Pavanetto0beed7055cde
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex b5d2585a8..caef504a0 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -161,4 +161,6 @@ class LxmlLinkExtractor(FilteringLinkExtractor):for doc in docs:links = self._extract_links(doc, response.url, response.encoding, base_url)all_links.extend(self._process_links(links))- return unique_list(all_links)+ if self.link_extractor.unique:+ return unique_list(all_links)+ return all_links