Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate unused scrapy utils (#6519)
Rohit Kumar Singhd2156696c45e
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 516218347..12c09839f 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -111,7 +111,7 @@ def md5sum(file: IO[bytes]) -> str:"""warnings.warn((- "The scrapy.utils.misc.md5sum function is deprecated, and will be "+ "The scrapy.utils.misc.md5sum function is deprecated and will be ""removed in a future version of Scrapy."),ScrapyDeprecationWarning,diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 6268af728..d970f5da5 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -8,12 +8,14 @@ import gcimport inspectimport reimport sys+import warningsimport weakreffrom collections.abc import AsyncIterable, Iterable, Mappingfrom functools import partial, wrapsfrom itertools import chainfrom typing import TYPE_CHECKING, Any, TypeVar, overload+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.asyncgen import as_async_generatorif TYPE_CHECKING:@@ -47,6 +49,11 @@ def flatten(x: Iterable[Any]) -> list[Any]:>>> flatten(["foo", ["baz", 42], "bar"])['foo', 'baz', 42, 'bar']"""+ warnings.warn(+ "The flatten function is deprecated and will be removed in a future version of Scrapy.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )return list(iflatten(x))@@ -54,6 +61,11 @@ def iflatten(x: Iterable[Any]) -> Iterable[Any]:"""iflatten(sequence) -> iteratorSimilar to ``.flatten()``, but returns iterator instead"""+ warnings.warn(+ "The iflatten function is deprecated and will be removed in a future version of Scrapy.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )for el in x:if is_listlike(el):yield from iflatten(el)@@ -272,6 +284,11 @@ def equal_attributes(obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None) -> bool:"""Compare two objects attributes"""+ warnings.warn(+ "The equal_attributes function is deprecated and will be removed in a future version of Scrapy.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )# not attributes given return False by defaultif not attributes:return Falsediff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 7848b9318..20e3151da 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -130,7 +130,7 @@ class RequestFingerprinter:if implementation != "SENTINEL":message = ("'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"- "And it will be removed in future version of Scrapy."+ "It will be removed in a future version of Scrapy.")warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)self._fingerprint = fingerprint@@ -147,6 +147,11 @@ def request_authenticate("""Authenticate the given request (in place) using the HTTP basic accessauthentication mechanism (RFC 2617) and the given username and password"""+ warnings.warn(+ "The request_authenticate function is deprecated and will be removed in a future version of Scrapy.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )request.headers["Authorization"] = basic_auth_header(username, password)diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.pyindex 3b4f67f00..308e351c6 100644--- a/scrapy/utils/serialize.py+++ b/scrapy/utils/serialize.py@@ -1,11 +1,13 @@import datetimeimport decimalimport json+import warningsfrom typing import Anyfrom itemadapter import ItemAdapter, is_itemfrom twisted.internet import defer+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Response@@ -36,4 +38,10 @@ class ScrapyJSONEncoder(json.JSONEncoder):class ScrapyJSONDecoder(json.JSONDecoder):- pass+ def __init__(self, *args, **kwargs):+ warnings.warn(+ "The ScrapyJSONDecoder class is deprecated and will be removed in a future version of Scrapy.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ super().__init__(*args, **kwargs)diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex d65f2a76d..92b73a91a 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -6,6 +6,7 @@ from __future__ import annotationsimport asyncioimport os+import warningsfrom importlib import import_modulefrom pathlib import Pathfrom posixpath import split@@ -16,6 +17,7 @@ from twisted.trial.unittest import SkipTestfrom scrapy import Spiderfrom scrapy.crawler import Crawler+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.boto import is_botocore_availableif TYPE_CHECKING:@@ -125,6 +127,11 @@ def assert_samelines("""Asserts text1 and text2 have the same lines, ignoring differences inline endings between platforms"""+ warnings.warn(+ "The assert_samelines function is deprecated and will be removed in a future version of Scrapy.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 5681ff9a4..f80f2517a 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -3,6 +3,7 @@ import operatorimport platformimport sys+import pytestfrom twisted.trial import unittestfrom scrapy.utils.asyncgen import as_async_generator, collect_asyncgen@@ -151,6 +152,7 @@ class BinaryIsTextTest(unittest.TestCase):class UtilsPythonTestCase(unittest.TestCase):+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_equal_attributes(self):class Obj:passdiff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 7156b13d0..965d050a4 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -6,6 +6,8 @@ import warningsfrom hashlib import sha1from weakref import WeakKeyDictionary+import pytest+from scrapy.http import Requestfrom scrapy.utils.python import to_bytesfrom scrapy.utils.request import (@@ -19,6 +21,7 @@ from scrapy.utils.test import get_crawlerclass UtilsRequestTest(unittest.TestCase):+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_request_authenticate(self):r = Request("http://www.example.com")request_authenticate(r, "someuser", "somepass")