Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate unused scrapy utils (#6519)

Rohit Kumar Singhd2156696c45e
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 516218347..12c09839f 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -111,7 +111,7 @@ def md5sum(file: IO[bytes]) -> str:
"""
warnings.warn(
(
- "The scrapy.utils.misc.md5sum function is deprecated, and will be "
+ "The scrapy.utils.misc.md5sum function is deprecated and will be "
"removed in a future version of Scrapy."
),
ScrapyDeprecationWarning,
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 6268af728..d970f5da5 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,12 +8,14 @@ import gc
import inspect
import re
import sys
+import warnings
import weakref
from collections.abc import AsyncIterable, Iterable, Mapping
from functools import partial, wraps
from itertools import chain
from typing import TYPE_CHECKING, Any, TypeVar, overload
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncgen import as_async_generator
if TYPE_CHECKING:
@@ -47,6 +49,11 @@ def flatten(x: Iterable[Any]) -> list[Any]:
>>> flatten(["foo", ["baz", 42], "bar"])
['foo', 'baz', 42, 'bar']
"""
+ warnings.warn(
+ "The flatten function is deprecated and will be removed in a future version of Scrapy.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
return list(iflatten(x))
@@ -54,6 +61,11 @@ def iflatten(x: Iterable[Any]) -> Iterable[Any]:
"""iflatten(sequence) -> iterator
Similar to ``.flatten()``, but returns iterator instead"""
+ warnings.warn(
+ "The iflatten function is deprecated and will be removed in a future version of Scrapy.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
for el in x:
if is_listlike(el):
yield from iflatten(el)
@@ -272,6 +284,11 @@ def equal_attributes(
obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None
) -> bool:
"""Compare two objects attributes"""
+ warnings.warn(
+ "The equal_attributes function is deprecated and will be removed in a future version of Scrapy.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
# not attributes given return False by default
if not attributes:
return False
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 7848b9318..20e3151da 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -130,7 +130,7 @@ class RequestFingerprinter:
if implementation != "SENTINEL":
message = (
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
- "And it will be removed in future version of Scrapy."
+ "It will be removed in a future version of Scrapy."
)
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
self._fingerprint = fingerprint
@@ -147,6 +147,11 @@ def request_authenticate(
"""Authenticate the given request (in place) using the HTTP basic access
authentication mechanism (RFC 2617) and the given username and password
"""
+ warnings.warn(
+ "The request_authenticate function is deprecated and will be removed in a future version of Scrapy.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
request.headers["Authorization"] = basic_auth_header(username, password)
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index 3b4f67f00..308e351c6 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -1,11 +1,13 @@
import datetime
import decimal
import json
+import warnings
from typing import Any
from itemadapter import ItemAdapter, is_item
from twisted.internet import defer
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
@@ -36,4 +38,10 @@ class ScrapyJSONEncoder(json.JSONEncoder):
class ScrapyJSONDecoder(json.JSONDecoder):
- pass
+ def __init__(self, *args, **kwargs):
+ warnings.warn(
+ "The ScrapyJSONDecoder class is deprecated and will be removed in a future version of Scrapy.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ super().__init__(*args, **kwargs)
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index d65f2a76d..92b73a91a 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -6,6 +6,7 @@ from __future__ import annotations
import asyncio
import os
+import warnings
from importlib import import_module
from pathlib import Path
from posixpath import split
@@ -16,6 +17,7 @@ from twisted.trial.unittest import SkipTest
from scrapy import Spider
from scrapy.crawler import Crawler
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore_available
if TYPE_CHECKING:
@@ -125,6 +127,11 @@ def assert_samelines(
"""Asserts text1 and text2 have the same lines, ignoring differences in
line endings between platforms
"""
+ warnings.warn(
+ "The assert_samelines function is deprecated and will be removed in a future version of Scrapy.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 5681ff9a4..f80f2517a 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -3,6 +3,7 @@ import operator
import platform
import sys
+import pytest
from twisted.trial import unittest
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
@@ -151,6 +152,7 @@ class BinaryIsTextTest(unittest.TestCase):
class UtilsPythonTestCase(unittest.TestCase):
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_equal_attributes(self):
class Obj:
pass
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 7156b13d0..965d050a4 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -6,6 +6,8 @@ import warnings
from hashlib import sha1
from weakref import WeakKeyDictionary
+import pytest
+
from scrapy.http import Request
from scrapy.utils.python import to_bytes
from scrapy.utils.request import (
@@ -19,6 +21,7 @@ from scrapy.utils.test import get_crawler
class UtilsRequestTest(unittest.TestCase):
+ @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_request_authenticate(self):
r = Request("http://www.example.com")
request_authenticate(r, "someuser", "somepass")