Scrapy, a fast high-level web crawling & scraping framework for Python.
Full typing for scrapy/pipelines. (#6387)
Andrey Rakhmatulline56b425198bf
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex f9544d329..0cfbc156f 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -10,6 +10,7 @@ from twisted.internet.defer import Deferredfrom scrapy import Spiderfrom scrapy.middleware import MiddlewareManager+from scrapy.settings import Settingsfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import deferred_f_from_coro_f@@ -18,7 +19,7 @@ class ItemPipelineManager(MiddlewareManager):component_name = "item pipeline"@classmethod- def _get_mwlist_from_settings(cls, settings) -> List[Any]:+ def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:return build_component_list(settings.getwithbase("ITEM_PIPELINES"))def _add_middleware(self, pipe: Any) -> None:diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex c1ce0939c..85a8c77da 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -18,16 +18,35 @@ from ftplib import FTPfrom io import BytesIOfrom os import PathLikefrom pathlib import Path-from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast+from typing import (+ IO,+ TYPE_CHECKING,+ Any,+ Callable,+ DefaultDict,+ Dict,+ List,+ NoReturn,+ Optional,+ Protocol,+ Set,+ Type,+ TypedDict,+ Union,+ cast,+)from urllib.parse import urlparsefrom itemadapter import ItemAdapterfrom twisted.internet import defer, threads+from twisted.internet.defer import Deferred+from twisted.python.failure import Failure+from scrapy import Spiderfrom scrapy.exceptions import IgnoreRequest, NotConfigured-from scrapy.http import Request+from scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACK-from scrapy.pipelines.media import MediaPipeline+from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipelinefrom scrapy.settings import Settingsfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.datatypes import CaseInsensitiveDict@@ -40,10 +59,11 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+logger = logging.getLogger(__name__)-def _to_string(path: Union[str, PathLike]) -> str:+def _to_string(path: Union[str, PathLike[str]]) -> str:return str(path) # convert a Path object to string@@ -68,23 +88,54 @@ class FileException(Exception):"""General media error exception"""+class StatInfo(TypedDict, total=False):+ checksum: str+ last_modified: float+++class FilesStoreProtocol(Protocol):+ def __init__(self, basedir: str): ...++ def persist_file(+ self,+ path: str,+ buf: BytesIO,+ info: MediaPipeline.SpiderInfo,+ meta: Optional[Dict[str, Any]] = None,+ headers: Optional[Dict[str, str]] = None,+ ) -> Optional[Deferred[Any]]: ...++ def stat_file(+ self, path: str, info: MediaPipeline.SpiderInfo+ ) -> Union[StatInfo, Deferred[StatInfo]]: ...++class FSFilesStore:- def __init__(self, basedir: Union[str, PathLike]):+ def __init__(self, basedir: Union[str, PathLike[str]]):basedir = _to_string(basedir)if "://" in basedir:basedir = basedir.split("://", 1)[1]- self.basedir = basedir+ self.basedir: str = basedirself._mkdir(Path(self.basedir))- self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set)+ self.created_directories: DefaultDict[MediaPipeline.SpiderInfo, Set[str]] = (+ defaultdict(set)+ )def persist_file(- self, path: Union[str, PathLike], buf, info, meta=None, headers=None- ):+ self,+ path: Union[str, PathLike[str]],+ buf: BytesIO,+ info: MediaPipeline.SpiderInfo,+ meta: Optional[Dict[str, Any]] = None,+ headers: Optional[Dict[str, str]] = None,+ ) -> None:absolute_path = self._get_filesystem_path(path)self._mkdir(absolute_path.parent, info)absolute_path.write_bytes(buf.getvalue())- def stat_file(self, path: Union[str, PathLike], info):+ def stat_file(+ self, path: Union[str, PathLike[str]], info: MediaPipeline.SpiderInfo+ ) -> StatInfo:absolute_path = self._get_filesystem_path(path)try:last_modified = absolute_path.stat().st_mtime@@ -96,12 +147,14 @@ class FSFilesStore:return {"last_modified": last_modified, "checksum": checksum}- def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path:+ def _get_filesystem_path(self, path: Union[str, PathLike[str]]) -> Path:path_comps = _to_string(path).split("/")return Path(self.basedir, *path_comps)- def _mkdir(self, dirname: Path, domain: Optional[str] = None):- seen = self.created_directories[domain] if domain else set()+ def _mkdir(+ self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None+ ) -> None:+ seen: Set[str] = self.created_directories[domain] if domain else set()if str(dirname) not in seen:if not dirname.exists():dirname.mkdir(parents=True)@@ -122,7 +175,7 @@ class S3FilesStore:"Cache-Control": "max-age=172800",}- def __init__(self, uri):+ def __init__(self, uri: str):if not is_botocore_available():raise NotConfigured("missing botocore library")import botocore.session@@ -142,8 +195,10 @@ class S3FilesStore:raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")self.bucket, self.prefix = uri[5:].split("/", 1)- def stat_file(self, path, info):- def _onsuccess(boto_key):+ def stat_file(+ self, path: str, info: MediaPipeline.SpiderInfo+ ) -> Deferred[StatInfo]:+ def _onsuccess(boto_key: Dict[str, Any]) -> StatInfo:checksum = boto_key["ETag"].strip('"')last_modified = boto_key["LastModified"]modified_stamp = time.mktime(last_modified.timetuple())@@ -151,13 +206,23 @@ class S3FilesStore:return self._get_boto_key(path).addCallback(_onsuccess)- def _get_boto_key(self, path):+ def _get_boto_key(self, path: str) -> Deferred[Dict[str, Any]]:key_name = f"{self.prefix}{path}"- return threads.deferToThread(- self.s3_client.head_object, Bucket=self.bucket, Key=key_name+ return cast(+ "Deferred[Dict[str, Any]]",+ threads.deferToThread(+ self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined]+ ),)- def persist_file(self, path, buf, info, meta=None, headers=None):+ def persist_file(+ self,+ path: str,+ buf: BytesIO,+ info: MediaPipeline.SpiderInfo,+ meta: Optional[Dict[str, Any]] = None,+ headers: Optional[Dict[str, str]] = None,+ ) -> Deferred[Any]:"""Upload file to S3 storage"""key_name = f"{self.prefix}{path}"buf.seek(0)@@ -165,7 +230,7 @@ class S3FilesStore:if headers:extra.update(self._headers_to_botocore_kwargs(headers))return threads.deferToThread(- self.s3_client.put_object,+ self.s3_client.put_object, # type: ignore[attr-defined]Bucket=self.bucket,Key=key_name,Body=buf,@@ -174,7 +239,7 @@ class S3FilesStore:**extra,)- def _headers_to_botocore_kwargs(self, headers):+ def _headers_to_botocore_kwargs(self, headers: Dict[str, Any]) -> Dict[str, Any]:"""Convert headers to botocore keyword arguments."""# This is required while we need to support both boto and botocore.mapping = CaseInsensitiveDict(@@ -206,7 +271,7 @@ class S3FilesStore:"X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation",})- extra = {}+ extra: Dict[str, Any] = {}for key, value in headers.items():try:kwarg = mapping[key]@@ -226,13 +291,13 @@ class GCSFilesStore:# Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.POLICY = None- def __init__(self, uri):+ def __init__(self, uri: str):from google.cloud import storageclient = storage.Client(project=self.GCS_PROJECT_ID)bucket, prefix = uri[5:].split("/", 1)self.bucket = client.bucket(bucket)- self.prefix = prefix+ self.prefix: str = prefixpermissions = self.bucket.test_iam_permissions(["storage.objects.get", "storage.objects.create"])@@ -248,8 +313,10 @@ class GCSFilesStore:{"bucket": bucket},)- def stat_file(self, path, info):- def _onsuccess(blob):+ def stat_file(+ self, path: str, info: MediaPipeline.SpiderInfo+ ) -> Deferred[StatInfo]:+ def _onsuccess(blob) -> StatInfo:if blob:checksum = base64.b64decode(blob.md5_hash).hex()last_modified = time.mktime(blob.updated.timetuple())@@ -257,19 +324,29 @@ class GCSFilesStore:return {}blob_path = self._get_blob_path(path)- return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(- _onsuccess+ return cast(+ Deferred[StatInfo],+ threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(+ _onsuccess+ ),)- def _get_content_type(self, headers):+ def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str:if headers and "Content-Type" in headers:return headers["Content-Type"]return "application/octet-stream"- def _get_blob_path(self, path):+ def _get_blob_path(self, path: str) -> str:return self.prefix + path- def persist_file(self, path, buf, info, meta=None, headers=None):+ def persist_file(+ self,+ path: str,+ buf: BytesIO,+ info: MediaPipeline.SpiderInfo,+ meta: Optional[Dict[str, Any]] = None,+ headers: Optional[Dict[str, str]] = None,+ ) -> Deferred[Any]:blob_path = self._get_blob_path(path)blob = self.bucket.blob(blob_path)blob.cache_control = self.CACHE_CONTROL@@ -283,22 +360,33 @@ class GCSFilesStore:class FTPFilesStore:- FTP_USERNAME = None- FTP_PASSWORD = None- USE_ACTIVE_MODE = None+ FTP_USERNAME: Optional[str] = None+ FTP_PASSWORD: Optional[str] = None+ USE_ACTIVE_MODE: Optional[bool] = None- def __init__(self, uri):+ def __init__(self, uri: str):if not uri.startswith("ftp://"):raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'")u = urlparse(uri)- self.port = u.port- self.host = u.hostname+ assert u.port+ assert u.hostname+ self.port: int = u.port+ self.host: str = u.hostnameself.port = int(u.port or 21)- self.username = u.username or self.FTP_USERNAME- self.password = u.password or self.FTP_PASSWORD- self.basedir = u.path.rstrip("/")+ assert self.FTP_USERNAME+ assert self.FTP_PASSWORD+ self.username: str = u.username or self.FTP_USERNAME+ self.password: str = u.password or self.FTP_PASSWORD+ self.basedir: str = u.path.rstrip("/")- def persist_file(self, path, buf, info, meta=None, headers=None):+ def persist_file(+ self,+ path: str,+ buf: BytesIO,+ info: MediaPipeline.SpiderInfo,+ meta: Optional[Dict[str, Any]] = None,+ headers: Optional[Dict[str, str]] = None,+ ) -> Deferred[Any]:path = f"{self.basedir}/{path}"return threads.deferToThread(ftp_store_file,@@ -311,8 +399,10 @@ class FTPFilesStore:use_active_mode=self.USE_ACTIVE_MODE,)- def stat_file(self, path, info):- def _stat_file(path):+ def stat_file(+ self, path: str, info: MediaPipeline.SpiderInfo+ ) -> Deferred[StatInfo]:+ def _stat_file(path: str) -> StatInfo:try:ftp = FTP()ftp.connect(self.host, self.port)@@ -328,7 +418,7 @@ class FTPFilesStore:except Exception:return {}- return threads.deferToThread(_stat_file, path)+ return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path))class FilesPipeline(MediaPipeline):@@ -350,20 +440,23 @@ class FilesPipeline(MediaPipeline):"""- MEDIA_NAME = "file"- EXPIRES = 90- STORE_SCHEMES = {+ MEDIA_NAME: str = "file"+ EXPIRES: int = 90+ STORE_SCHEMES: Dict[str, Type[FilesStoreProtocol]] = {"": FSFilesStore,"file": FSFilesStore,"s3": S3FilesStore,"gs": GCSFilesStore,"ftp": FTPFilesStore,}- DEFAULT_FILES_URLS_FIELD = "file_urls"- DEFAULT_FILES_RESULT_FIELD = "files"+ DEFAULT_FILES_URLS_FIELD: str = "file_urls"+ DEFAULT_FILES_RESULT_FIELD: str = "files"def __init__(- self, store_uri: Union[str, PathLike], download_func=None, settings=None+ self,+ store_uri: Union[str, PathLike[str]],+ download_func: Optional[Callable[[Request, Spider], Response]] = None,+ settings: Union[Settings, Dict[str, Any], None] = None,):store_uri = _to_string(store_uri)if not store_uri:@@ -372,26 +465,26 @@ class FilesPipeline(MediaPipeline):if isinstance(settings, dict) or settings is None:settings = Settings(settings)cls_name = "FilesPipeline"- self.store = self._get_store(store_uri)+ self.store: FilesStoreProtocol = self._get_store(store_uri)resolve = functools.partial(self._key_for_pipe, base_class_name=cls_name, settings=settings)- self.expires = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)+ self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)if not hasattr(self, "FILES_URLS_FIELD"):self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELDif not hasattr(self, "FILES_RESULT_FIELD"):self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD- self.files_urls_field = settings.get(+ self.files_urls_field: str = settings.get(resolve("FILES_URLS_FIELD"), self.FILES_URLS_FIELD)- self.files_result_field = settings.get(+ self.files_result_field: str = settings.get(resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD)super().__init__(download_func=download_func, settings=settings)@classmethod- def from_settings(cls, settings) -> Self:+ def from_settings(cls, settings: Settings) -> Self:s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"])s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]@@ -418,7 +511,7 @@ class FilesPipeline(MediaPipeline):store_uri = settings["FILES_STORE"]return cls(store_uri, settings=settings)- def _get_store(self, uri: str):+ def _get_store(self, uri: str) -> FilesStoreProtocol:if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dirscheme = "file"else:@@ -426,19 +519,21 @@ class FilesPipeline(MediaPipeline):store_cls = self.STORE_SCHEMES[scheme]return store_cls(uri)- def media_to_download(self, request, info, *, item=None):- def _onsuccess(result):+ def media_to_download(+ self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None+ ) -> Deferred[Optional[FileInfo]]:+ def _onsuccess(result: StatInfo) -> Optional[FileInfo]:if not result:- return # returning None force download+ return None # returning None force downloadlast_modified = result.get("last_modified", None)if not last_modified:- return # returning None force download+ return None # returning None force downloadage_seconds = time.time() - last_modifiedage_days = age_seconds / 60 / 60 / 24if age_days > self.expires:- return # returning None force download+ return None # returning None force downloadreferer = referer_str(request)logger.debug(@@ -458,19 +553,22 @@ class FilesPipeline(MediaPipeline):}path = self.file_path(request, info=info, item=item)- dfd = defer.maybeDeferred(self.store.stat_file, path, info)- dfd.addCallback(_onsuccess)- dfd.addErrback(lambda _: None)- dfd.addErrback(+ # defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type+ dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type]+ dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess)+ dfd2.addErrback(lambda _: None)+ dfd2.addErrback(lambda f: logger.error(self.__class__.__name__ + ".store.stat_file",exc_info=failure_to_exc_info(f),extra={"spider": info.spider},))- return dfd+ return dfd2- def media_failed(self, failure, request, info):+ def media_failed(+ self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo+ ) -> NoReturn:if not isinstance(failure.value, IgnoreRequest):referer = referer_str(request)logger.warning(@@ -487,7 +585,14 @@ class FilesPipeline(MediaPipeline):raise FileException- def media_downloaded(self, response, request, info, *, item=None):+ def media_downloaded(+ self,+ response: Response,+ request: Request,+ info: MediaPipeline.SpiderInfo,+ *,+ item: Any = None,+ ) -> FileInfo:referer = referer_str(request)if response.status != 200:@@ -546,16 +651,26 @@ class FilesPipeline(MediaPipeline):"status": status,}- def inc_stats(self, spider, status):+ def inc_stats(self, spider: Spider, status: str) -> None:+ assert spider.crawler.statsspider.crawler.stats.inc_value("file_count", spider=spider)spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider)# Overridable Interface- def get_media_requests(self, item, info):+ def get_media_requests(+ self, item: Any, info: MediaPipeline.SpiderInfo+ ) -> List[Request]:urls = ItemAdapter(item).get(self.files_urls_field, [])return [Request(u, callback=NO_CALLBACK) for u in urls]- def file_downloaded(self, response, request, info, *, item=None):+ def file_downloaded(+ self,+ response: Response,+ request: Request,+ info: MediaPipeline.SpiderInfo,+ *,+ item: Any = None,+ ) -> str:path = self.file_path(request, response=response, info=info, item=item)buf = BytesIO(response.body)checksum = _md5sum(buf)@@ -563,12 +678,21 @@ class FilesPipeline(MediaPipeline):self.store.persist_file(path, buf, info)return checksum- def item_completed(self, results, item, info):+ def item_completed(+ self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo+ ) -> Any:with suppress(KeyError):ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]return item- def file_path(self, request, response=None, info=None, *, item=None):+ def file_path(+ self,+ request: Request,+ response: Optional[Response] = None,+ info: Optional[MediaPipeline.SpiderInfo] = None,+ *,+ item: Any = None,+ ) -> str:media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosecmedia_ext = Path(request.url).suffix# Handles empty and wild extensions by trying to guess the@@ -577,5 +701,5 @@ class FilesPipeline(MediaPipeline):media_ext = ""media_type = mimetypes.guess_type(request.url)[0]if media_type:- media_ext = mimetypes.guess_extension(media_type)+ media_ext = cast(str, mimetypes.guess_extension(media_type))return f"full/{media_guid}{media_ext}"diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex e7ef06fb3..27a57b17c 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -12,12 +12,25 @@ import warningsfrom contextlib import suppressfrom io import BytesIOfrom os import PathLike-from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast+from typing import (+ TYPE_CHECKING,+ Any,+ Callable,+ Dict,+ Iterable,+ List,+ Optional,+ Tuple,+ Type,+ Union,+ cast,+)from itemadapter import ItemAdapter+from scrapy import Spiderfrom scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning-from scrapy.http import Request+from scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import (FileException,@@ -27,20 +40,20 @@ from scrapy.pipelines.files import (S3FilesStore,_md5sum,)--# TODO: from scrapy.pipelines.media import MediaPipeline+from scrapy.pipelines.media import FileInfoOrError, MediaPipelinefrom scrapy.settings import Settingsfrom scrapy.utils.python import get_func_args, to_bytesif TYPE_CHECKING:# typing.Self requires Python 3.11+ from PIL import Imagefrom typing_extensions import Selfclass NoimagesDrop(DropItem):"""Product with no images exception"""- def __init__(self, *args, **kwargs):+ def __init__(self, *args: Any, **kwargs: Any):warnings.warn("The NoimagesDrop class is deprecated",category=ScrapyDeprecationWarning,@@ -56,19 +69,22 @@ class ImageException(FileException):class ImagesPipeline(FilesPipeline):"""Abstract pipeline that implement the image thumbnail generation logic"""- MEDIA_NAME = "image"+ MEDIA_NAME: str = "image"# Uppercase attributes kept for backward compatibility with code that subclasses# ImagesPipeline. They may be overridden by settings.- MIN_WIDTH = 0- MIN_HEIGHT = 0- EXPIRES = 90+ MIN_WIDTH: int = 0+ MIN_HEIGHT: int = 0+ EXPIRES: int = 90THUMBS: Dict[str, Tuple[int, int]] = {}DEFAULT_IMAGES_URLS_FIELD = "image_urls"DEFAULT_IMAGES_RESULT_FIELD = "images"def __init__(- self, store_uri: Union[str, PathLike], download_func=None, settings=None+ self,+ store_uri: Union[str, PathLike[str]],+ download_func: Optional[Callable[[Request, Spider], Response]] = None,+ settings: Union[Settings, Dict[str, Any], None] = None,):try:from PIL import Image@@ -89,27 +105,33 @@ class ImagesPipeline(FilesPipeline):base_class_name="ImagesPipeline",settings=settings,)- self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)+ self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)if not hasattr(self, "IMAGES_RESULT_FIELD"):- self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD+ self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELDif not hasattr(self, "IMAGES_URLS_FIELD"):- self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD+ self.IMAGES_URLS_FIELD: str = self.DEFAULT_IMAGES_URLS_FIELD- self.images_urls_field = settings.get(+ self.images_urls_field: str = settings.get(resolve("IMAGES_URLS_FIELD"), self.IMAGES_URLS_FIELD)- self.images_result_field = settings.get(+ self.images_result_field: str = settings.get(resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD)- self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH)- self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT)- self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS)+ self.min_width: int = settings.getint(+ resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH+ )+ self.min_height: int = settings.getint(+ resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT+ )+ self.thumbs: Dict[str, Tuple[int, int]] = settings.get(+ resolve("IMAGES_THUMBS"), self.THUMBS+ )- self._deprecated_convert_image = None+ self._deprecated_convert_image: Optional[bool] = None@classmethod- def from_settings(cls, settings) -> Self:+ def from_settings(cls, settings: Settings) -> Self:s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"])s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]@@ -136,11 +158,25 @@ class ImagesPipeline(FilesPipeline):store_uri = settings["IMAGES_STORE"]return cls(store_uri, settings=settings)- def file_downloaded(self, response, request, info, *, item=None):+ def file_downloaded(+ self,+ response: Response,+ request: Request,+ info: MediaPipeline.SpiderInfo,+ *,+ item: Any = None,+ ) -> str:return self.image_downloaded(response, request, info, item=item)- def image_downloaded(self, response, request, info, *, item=None):- checksum = None+ def image_downloaded(+ self,+ response: Response,+ request: Request,+ info: MediaPipeline.SpiderInfo,+ *,+ item: Any = None,+ ) -> str:+ checksum: Optional[str] = Nonefor path, image, buf in self.get_images(response, request, info, item=item):if checksum is None:buf.seek(0)@@ -153,9 +189,17 @@ class ImagesPipeline(FilesPipeline):meta={"width": width, "height": height},headers={"Content-Type": "image/jpeg"},)+ assert checksum is not Nonereturn checksum- def get_images(self, response, request, info, *, item=None):+ def get_images(+ self,+ response: Response,+ request: Request,+ info: MediaPipeline.SpiderInfo,+ *,+ item: Any = None,+ ) -> Iterable[Tuple[str, Image.Image, BytesIO]]:path = self.file_path(request, response=response, info=info, item=item)orig_image = self._Image.open(BytesIO(response.body))@@ -196,7 +240,12 @@ class ImagesPipeline(FilesPipeline):thumb_image, thumb_buf = self.convert_image(image, size, buf)yield thumb_path, thumb_image, thumb_buf- def convert_image(self, image, size=None, response_body=None):+ def convert_image(+ self,+ image: Image.Image,+ size: Optional[Tuple[int, int]] = None,+ response_body: Optional[BytesIO] = None,+ ) -> Tuple[Image.Image, BytesIO]:if response_body is None:warnings.warn(f"{self.__class__.__name__}.convert_image() method called in a deprecated way, "@@ -225,7 +274,7 @@ class ImagesPipeline(FilesPipeline):# when updating the minimum requirements for Pillow.resampling_filter = self._Image.Resampling.LANCZOSexcept AttributeError:- resampling_filter = self._Image.ANTIALIAS+ resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]image.thumbnail(size, resampling_filter)elif response_body is not None and image.format == "JPEG":return image, response_body@@ -234,19 +283,38 @@ class ImagesPipeline(FilesPipeline):image.save(buf, "JPEG")return image, buf- def get_media_requests(self, item, info):+ def get_media_requests(+ self, item: Any, info: MediaPipeline.SpiderInfo+ ) -> List[Request]:urls = ItemAdapter(item).get(self.images_urls_field, [])return [Request(u, callback=NO_CALLBACK) for u in urls]- def item_completed(self, results, item, info):+ def item_completed(+ self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo+ ) -> Any:with suppress(KeyError):ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]return item- def file_path(self, request, response=None, info=None, *, item=None):+ def file_path(+ self,+ request: Request,+ response: Optional[Response] = None,+ info: Optional[MediaPipeline.SpiderInfo] = None,+ *,+ item: Any = None,+ ) -> str:image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosecreturn f"full/{image_guid}.jpg"- def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):+ def thumb_path(+ self,+ request: Request,+ thumb_id: str,+ response: Optional[Response] = None,+ info: Optional[MediaPipeline.SpiderInfo] = None,+ *,+ item: Any = None,+ ) -> str:thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosecreturn f"thumbs/{thumb_id}/{thumb_guid}.jpg"diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 0e374265e..3e327105e 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -4,54 +4,101 @@ import functoolsimport loggingfrom abc import ABC, abstractmethodfrom collections import defaultdict-from typing import TYPE_CHECKING+from typing import (+ TYPE_CHECKING,+ Any,+ Callable,+ DefaultDict,+ Dict,+ List,+ Literal,+ NoReturn,+ Optional,+ Set,+ Tuple,+ TypedDict,+ TypeVar,+ Union,+ cast,+)from twisted.internet.defer import Deferred, DeferredListfrom twisted.python.failure import Failure-from scrapy.http.request import NO_CALLBACK+from scrapy import Spider+from scrapy.crawler import Crawler+from scrapy.http import Response+from scrapy.http.request import NO_CALLBACK, Requestfrom scrapy.settings import Settingsfrom scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.defer import defer_result, mustbe_deferredfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import arg_to_iter+from scrapy.utils.request import RequestFingerprinterif TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+_T = TypeVar("_T")+++class FileInfo(TypedDict):+ url: str+ path: str+ checksum: Optional[str]+ status: str+++FileInfoOrError = Union[Tuple[Literal[True], FileInfo], Tuple[Literal[False], Failure]]+logger = logging.getLogger(__name__)class MediaPipeline(ABC):- LOG_FAILED_RESULTS = True+ crawler: Crawler+ _fingerprinter: RequestFingerprinter- class SpiderInfo:- def __init__(self, spider):- self.spider = spider- self.downloading = set()- self.downloaded = {}- self.waiting = defaultdict(list)+ LOG_FAILED_RESULTS: bool = True- def __init__(self, download_func=None, settings=None):+ class SpiderInfo:+ def __init__(self, spider: Spider):+ self.spider: Spider = spider+ self.downloading: Set[bytes] = set()+ self.downloaded: Dict[bytes, Union[FileInfo, Failure]] = {}+ self.waiting: DefaultDict[bytes, List[Deferred[FileInfo]]] = defaultdict(+ list+ )++ def __init__(+ self,+ download_func: Optional[Callable[[Request, Spider], Response]] = None,+ settings: Union[Settings, Dict[str, Any], None] = None,+ ):self.download_func = download_func- self._expects_item = {}if isinstance(settings, dict) or settings is None:settings = Settings(settings)resolve = functools.partial(self._key_for_pipe, base_class_name="MediaPipeline", settings=settings)- self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False)+ self.allow_redirects: bool = settings.getbool(+ resolve("MEDIA_ALLOW_REDIRECTS"), False+ )self._handle_statuses(self.allow_redirects)- def _handle_statuses(self, allow_redirects):+ def _handle_statuses(self, allow_redirects: bool) -> None:self.handle_httpstatus_list = Noneif allow_redirects:self.handle_httpstatus_list = SequenceExclude(range(300, 400))- def _key_for_pipe(self, key, base_class_name=None, settings=None):+ def _key_for_pipe(+ self,+ key: str,+ base_class_name: Optional[str] = None,+ settings: Optional[Settings] = None,+ ) -> str:class_name = self.__class__.__name__formatted_key = f"{class_name.upper()}_{key}"if (@@ -64,26 +111,34 @@ class MediaPipeline(ABC):return formatted_key@classmethod- def from_crawler(cls, crawler) -> Self:+ def from_crawler(cls, crawler: Crawler) -> Self:+ pipe: Selftry:pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined]except AttributeError:pipe = cls()pipe.crawler = crawler+ assert crawler.request_fingerprinterpipe._fingerprinter = crawler.request_fingerprinterreturn pipe- def open_spider(self, spider):+ def open_spider(self, spider: Spider) -> None:self.spiderinfo = self.SpiderInfo(spider)- def process_item(self, item, spider):+ def process_item(+ self, item: Any, spider: Spider+ ) -> Deferred[List[FileInfoOrError]]:info = self.spiderinforequests = arg_to_iter(self.get_media_requests(item, info))dlist = [self._process_request(r, info, item) for r in requests]- dfd = DeferredList(dlist, consumeErrors=True)+ dfd = cast(+ "Deferred[List[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True)+ )return dfd.addCallback(self.item_completed, item, info)- def _process_request(self, request, info, item):+ def _process_request(+ self, request: Request, info: SpiderInfo, item: Any+ ) -> Deferred[FileInfo]:fp = self._fingerprinter.fingerprint(request)eb = request.errbackrequest.callback = NO_CALLBACK@@ -97,7 +152,7 @@ class MediaPipeline(ABC):return d# Otherwise, wait for result- wad = Deferred()+ wad: Deferred[FileInfo] = Deferred()if eb:wad.addErrback(eb)info.waiting[fp].append(wad)@@ -108,36 +163,48 @@ class MediaPipeline(ABC):# Download request checking media_to_download hook output firstinfo.downloading.add(fp)- dfd = mustbe_deferred(self.media_to_download, request, info, item=item)- dfd.addCallback(self._check_media_to_download, request, info, item=item)- dfd.addErrback(self._log_exception)- dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)- return dfd.addBoth(lambda _: wad) # it must return wad at last+ dfd: Deferred[Optional[FileInfo]] = mustbe_deferred(+ self.media_to_download, request, info, item=item+ )+ dfd2: Deferred[FileInfo] = dfd.addCallback(+ self._check_media_to_download, request, info, item=item+ )+ dfd2.addErrback(self._log_exception)+ dfd2.addBoth(self._cache_result_and_execute_waiters, fp, info)+ return dfd2.addBoth(lambda _: wad) # it must return wad at last- def _log_exception(self, result):+ def _log_exception(self, result: Failure) -> Failure:logger.exception(result)return result- def _modify_media_request(self, request):+ def _modify_media_request(self, request: Request) -> None:if self.handle_httpstatus_list:request.meta["handle_httpstatus_list"] = self.handle_httpstatus_listelse:request.meta["handle_httpstatus_all"] = True- def _check_media_to_download(self, result, request, info, item):+ def _check_media_to_download(+ self, result: Optional[FileInfo], request: Request, info: SpiderInfo, item: Any+ ) -> Union[FileInfo, Deferred[FileInfo]]:if result is not None:return result+ dfd: Deferred[Response]if self.download_func:# this ugly code was left only to support tests. TODO: removedfd = mustbe_deferred(self.download_func, request, info.spider)else:self._modify_media_request(request)+ assert self.crawler.enginedfd = self.crawler.engine.download(request)- dfd.addCallback(self.media_downloaded, request, info, item=item)- dfd.addErrback(self.media_failed, request, info)- return dfd+ dfd2: Deferred[FileInfo] = dfd.addCallback(+ self.media_downloaded, request, info, item=item+ )+ dfd2.addErrback(self.media_failed, request, info)+ return dfd2- def _cache_result_and_execute_waiters(self, result, fp, info):+ def _cache_result_and_execute_waiters(+ self, result: Union[FileInfo, Failure], fp: bytes, info: SpiderInfo+ ) -> None:if isinstance(result, Failure):# minimize cached information for failureresult.cleanFailure()@@ -176,30 +243,44 @@ class MediaPipeline(ABC):# Overridable Interface@abstractmethod- def media_to_download(self, request, info, *, item=None):+ def media_to_download(+ self, request: Request, info: SpiderInfo, *, item: Any = None+ ) -> Deferred[Optional[FileInfo]]:"""Check request before starting download"""raise NotImplementedError()@abstractmethod- def get_media_requests(self, item, info):+ def get_media_requests(self, item: Any, info: SpiderInfo) -> List[Request]:"""Returns the media requests to download"""raise NotImplementedError()@abstractmethod- def media_downloaded(self, response, request, info, *, item=None):+ def media_downloaded(+ self,+ response: Response,+ request: Request,+ info: SpiderInfo,+ *,+ item: Any = None,+ ) -> FileInfo:"""Handler for success downloads"""raise NotImplementedError()@abstractmethod- def media_failed(self, failure, request, info):+ def media_failed(+ self, failure: Failure, request: Request, info: SpiderInfo+ ) -> NoReturn:"""Handler for failed downloads"""raise NotImplementedError()- def item_completed(self, results, item, info):+ def item_completed(+ self, results: List[FileInfoOrError], item: Any, info: SpiderInfo+ ) -> Any:"""Called per item when all media requests has been processed"""if self.LOG_FAILED_RESULTS:for ok, value in results:if not ok:+ assert isinstance(value, Failure)logger.error("%(class)s found errors processing %(item)s",{"class": self.__class__.__name__, "item": item},@@ -209,6 +290,13 @@ class MediaPipeline(ABC):return item@abstractmethod- def file_path(self, request, response=None, info=None, *, item=None):+ def file_path(+ self,+ request: Request,+ response: Optional[Response] = None,+ info: Optional[SpiderInfo] = None,+ *,+ item: Any = None,+ ) -> str:"""Returns the path where downloaded media should be stored"""raise NotImplementedError()