Scrapy, a fast high-level web crawling & scraping framework for Python.
Provide complete API documentation coverage of scrapy.linkextractors
Adrián Chaves7f4f98fd38d3
diff --git a/docs/conf.py b/docs/conf.pyindex 34dd5bcb7..5ba6b8d43 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -265,6 +265,10 @@ coverage_ignore_pyobjects = [# Never documented before, and deprecated now.r'^scrapy\.item\.DictItem$',+ r'^scrapy\.linkextractors\.FilteringLinkExtractor$',++ # Implementation detail of LxmlLinkExtractor+ r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor',]diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rstindex 713a94e10..f9936a498 100644--- a/docs/topics/link-extractors.rst+++ b/docs/topics/link-extractors.rst@@ -4,46 +4,33 @@Link Extractors===============-Link extractors are objects whose only purpose is to extract links from web-pages (:class:`scrapy.http.Response` objects) which will be eventually-followed.+A link extractor is an object that extracts links from responses.-There is ``scrapy.linkextractors.LinkExtractor`` available-in Scrapy, but you can create your own custom Link Extractors to suit your-needs by implementing a simple interface.--The only public method that every link extractor has is ``extract_links``,-which receives a :class:`~scrapy.http.Response` object and returns a list-of :class:`scrapy.link.Link` objects. Link extractors are meant to be-instantiated once and their ``extract_links`` method called several times-with different responses to extract links to follow.--Link extractors are used in the :class:`~scrapy.spiders.CrawlSpider`-class (available in Scrapy), through a set of rules, but you can also use it in-your spiders, even if you don't subclass from-:class:`~scrapy.spiders.CrawlSpider`, as its purpose is very simple: to-extract links.+The constructor of :class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`+takes settings that determine which links may be extracted.+:class:`LxmlLinkExtractor.extract_links+<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a+list of matching :class:`scrapy.link.Link` objects from a+:class:`~scrapy.http.Response` object.+Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders+through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link+extractors in regular spiders... _topics-link-extractors-ref:-Built-in link extractors reference-==================================+Link extractor reference+========================.. module:: scrapy.linkextractors:synopsis: Link extractors classes-Link extractors classes bundled with Scrapy are provided in the-:mod:`scrapy.linkextractors` module.--The default link extractor is ``LinkExtractor``, which is the same as-:class:`~.LxmlLinkExtractor`::+The link extractor class is+:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it+can also be imported as ``scrapy.linkextractors.LinkExtractor``::from scrapy.linkextractors import LinkExtractor-There used to be other link extractor classes in previous Scrapy versions,-but they are deprecated now.-LxmlLinkExtractor-----------------@@ -152,4 +139,6 @@ LxmlLinkExtractorfrom elements or attributes which allow leading/trailing whitespaces).:type strip: boolean+ .. automethod:: extract_links+.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.pydiff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.pyindex ebf3cd7d8..ca80dc339 100644--- a/scrapy/linkextractors/__init__.py+++ b/scrapy/linkextractors/__init__.py@@ -6,11 +6,13 @@ This package contains a collection of Link Extractors.For more info see docs/topics/link-extractors.rst"""import re+from warnings import warnfrom six.moves.urllib.parse import urlparsefrom parsel.csstranslator import HTMLTranslatorfrom w3lib.url import canonicalize_url+from scrapy.utils.deprecate import ScrapyDeprecationWarningfrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.url import (url_is_from_any_domain, url_has_any_extension,@@ -49,6 +51,15 @@ class FilteringLinkExtractor(object):_csstranslator = HTMLTranslator()+ def __new__(cls, *args, **kwargs):+ from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor+ if (issubclass(cls, FilteringLinkExtractor) and+ not issubclass(cls, LxmlLinkExtractor)):+ warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '+ 'please use scrapy.linkextractors.LinkExtractor instead',+ ScrapyDeprecationWarning, stacklevel=2)+ return super(FilteringLinkExtractor, cls).__new__(cls)+def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 8f6f93a44..41091ba23 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -117,6 +117,14 @@ class LxmlLinkExtractor(FilteringLinkExtractor):restrict_text=restrict_text)def extract_links(self, response):+ """Returns a list of :class:`~scrapy.link.Link` objects from the+ specified :class:`response <scrapy.http.Response>`.++ Only links that match the settings passed to the link extractor+ constructor are returned.++ Duplicate links are omitted.+ """base_url = get_base_url(response)if self.restrict_xpaths:docs = [subdocdiff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex d96e259f6..ea6db28c0 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -1,10 +1,13 @@import reimport unittest+from warnings import catch_warningsimport pytest+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, XmlResponsefrom scrapy.link import Link+from scrapy.linkextractors import FilteringLinkExtractorfrom scrapy.linkextractors.lxmlhtml import LxmlLinkExtractorfrom tests import get_testdata@@ -506,3 +509,30 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):@pytest.mark.xfaildef test_restrict_xpaths_with_html_entities(self):super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()++ def test_filteringlinkextractor_deprecation_warning(self):+ """Make sure the FilteringLinkExtractor deprecation warning is not+ issued for LxmlLinkExtractor"""+ with catch_warnings(record=True) as warnings:+ extractor = LxmlLinkExtractor()+ self.assertEqual(len(warnings), 0)+ class SubclassedItem(LxmlLinkExtractor):+ pass+ subclassed_extractor = SubclassedItem()+ self.assertEqual(len(warnings), 0)+++class FilteringLinkExtractorTest(unittest.TestCase):++ def test_deprecation_warning(self):+ args = [None]*10+ with catch_warnings(record=True) as warnings:+ extractor = FilteringLinkExtractor(*args)+ self.assertEqual(len(warnings), 1)+ self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)+ with catch_warnings(record=True) as warnings:+ class SubclassedFilteringLinkExtractor(FilteringLinkExtractor):+ pass+ subclassed_extractor = SubclassedFilteringLinkExtractor(*args)+ self.assertEqual(len(warnings), 1)+ self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)