Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

More granular untyped defs checking in tests. (#7712)

Andrey Rakhmatullin61f99f2df1d3
diff --git a/pyproject.toml b/pyproject.toml
index ed2e99bea..ac9d37e78 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -94,7 +94,82 @@ untyped_calls_exclude = [
[[tool.mypy.overrides]]
module = "tests.*"
allow_untyped_defs = true
-allow_incomplete_defs = true # 48 errors
+allow_incomplete_defs = true # 59 errors
+
+# TODO
+[[tool.mypy.overrides]]
+module = [
+ "tests.mockserver.*",
+ "tests.spiders",
+ "tests.test_closespider",
+ "tests.test_cmdline",
+ "tests.test_contracts",
+ "tests.test_core_downloader",
+ "tests.test_downloader_handler_twisted_ftp",
+ "tests.test_downloadermiddleware_cookies",
+ "tests.test_downloadermiddleware_httpauth",
+ "tests.test_downloadermiddleware_httpcache",
+ "tests.test_downloadermiddleware_httpcompression",
+ "tests.test_downloadermiddleware_httpproxy",
+ "tests.test_downloadermiddleware_offsite",
+ "tests.test_downloadermiddleware_redirect",
+ "tests.test_downloadermiddleware_redirect_base",
+ "tests.test_downloadermiddleware_redirect_metarefresh",
+ "tests.test_downloadermiddleware_retry",
+ "tests.test_downloadermiddleware_robotstxt",
+ "tests.test_downloadermiddleware_stats",
+ "tests.test_downloaderslotssettings",
+ "tests.test_dupefilters",
+ "tests.test_engine_loop",
+ "tests.test_exporters",
+ "tests.test_extension_statsmailer",
+ "tests.test_extension_throttle",
+ "tests.test_feedexport",
+ "tests.test_feedexport_postprocess",
+ "tests.test_feedexport_storages",
+ "tests.test_feedexport_uri_params",
+ "tests.test_http2_client_protocol",
+ "tests.test_http_headers",
+ "tests.test_http_request",
+ "tests.test_http_request_form",
+ "tests.test_http_response",
+ "tests.test_http_response_text",
+ "tests.test_item",
+ "tests.test_link",
+ "tests.test_linkextractors",
+ "tests.test_loader",
+ "tests.test_logformatter",
+ "tests.test_logstats",
+ "tests.test_mail",
+ "tests.test_pipeline_crawl",
+ "tests.test_pipeline_files",
+ "tests.test_pipeline_images",
+ "tests.test_pipeline_media",
+ "tests.test_pipelines",
+ "tests.test_pqueues",
+ "tests.test_request_attribute_binding",
+ "tests.test_request_cb_kwargs",
+ "tests.test_request_dict",
+ "tests.test_request_left",
+ "tests.test_robotstxt_interface",
+ "tests.test_scheduler_base",
+ "tests.test_settings",
+ "tests.test_spider",
+ "tests.test_spider_crawl",
+ "tests.test_spidermiddleware_output_chain",
+ "tests.test_spidermiddleware_process_start",
+ "tests.test_spider_sitemap",
+ "tests.test_squeues",
+ "tests.test_squeues_request",
+ "tests.test_stats",
+ "tests.test_utils_datatypes",
+ "tests.test_utils_decorators",
+ "tests.test_utils_defer",
+ "tests.test_utils_deprecate",
+ "tests.test_utils_misc.test_return_with_argument_inside_generator",
+ "tests.test_utils_python",
+ "tests.test_utils_request",
+]
check_untyped_defs = false
# Interface classes are hard to support
diff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
index e000a23db..2f4f2d051 100644
--- a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
@@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro
class UppercasePipeline:
- async def _open_spider(self, spider):
+ async def _open_spider(self, spider: Spider) -> None:
spider.logger.info("async pipeline opened!")
await asyncio.sleep(0.1)
diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py
index a6520ca22..7e3988d92 100644
--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py
+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py
@@ -10,6 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):
"""
name = "caching_hostname_resolver_spider"
+ url: str
async def start(self):
yield scrapy.Request(self.url)
diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py
index eb2e1105c..6e4db1990 100644
--- a/tests/CrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro
class UppercasePipeline:
- async def _open_spider(self, spider):
+ async def _open_spider(self, spider: Spider) -> None:
spider.logger.info("async pipeline opened!")
await asyncio.sleep(0.1)
diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py
index 7f687e7d9..79ce9c707 100644
--- a/tests/CrawlerProcess/caching_hostname_resolver.py
+++ b/tests/CrawlerProcess/caching_hostname_resolver.py
@@ -10,6 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):
"""
name = "caching_hostname_resolver_spider"
+ url: str
async def start(self):
yield scrapy.Request(self.url)
diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py
index a577b1909..9f7d14606 100644
--- a/tests/CrawlerProcess/sleeping.py
+++ b/tests/CrawlerProcess/sleeping.py
@@ -15,7 +15,7 @@ class SleepingSpider(scrapy.Spider):
async def parse(self, response):
from twisted.internet import reactor
- d = Deferred()
+ d: Deferred[None] = Deferred()
reactor.callLater(int(sys.argv[1]), d.callback, None)
await maybe_deferred_to_future(d)
diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index fae206610..bab4f65db 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -36,6 +36,7 @@ def createResolver(servers: list[tuple[str, int]]) -> ResolverBase:
class LocalhostSpider(Spider):
name = "localhost_spider"
+ url: str
async def start(self):
yield Request(self.url)
diff --git a/tests/test_addons.py b/tests/test_addons.py
index 14ebddda8..a041af6d6 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -107,11 +107,11 @@ class TestAddonManager:
addonlist = []
for i in range(3):
addon = get_addon_cls({"KEY1": i})
- addon.number = i
+ addon.number = i # type: ignore[attr-defined]
addonlist.append(addon)
# Test for every possible ordering
for ordered_addons in itertools.permutations(addonlist):
- expected_order = [a.number for a in ordered_addons]
+ expected_order = [a.number for a in ordered_addons] # type: ignore[attr-defined]
settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}
crawler = get_crawler(settings_dict=settings)
manager = crawler.addons
@@ -177,7 +177,7 @@ class TestAddonManager:
)
settings["SCHEDULER"] = "AddonScheduler"
- settings_dict = {
+ settings_dict: dict[str, Any] = {
"ADDONS": {AddonWithFallback: 1},
}
crawler = get_crawler(settings_dict=settings_dict)
diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py
index f17543575..50b04de0b 100644
--- a/tests/test_cmdline_crawl_with_pipeline/__init__.py
+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py
@@ -1,21 +1,24 @@
+from __future__ import annotations
+
import sys
from pathlib import Path
from subprocess import PIPE, Popen
-class TestCmdlineCrawlPipeline:
- def _execute(self, spname):
- args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)
- cwd = Path(__file__).resolve().parent
- proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
- _, stderr = proc.communicate()
- return proc.returncode, stderr
+def _execute(spname: str) -> int:
+ args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)
+ cwd = Path(__file__).resolve().parent
+ proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
+ proc.communicate()
+ return proc.returncode
+
+
+def test_open_spider_normally_in_pipeline():
+ returncode = _execute("normal")
+ assert returncode == 0
- def test_open_spider_normally_in_pipeline(self):
- returncode, _ = self._execute("normal")
- assert returncode == 0
- def test_exception_at_open_spider_in_pipeline(self):
- returncode, _ = self._execute("exception")
- # An exception in pipeline's open_spider should result in a non-zero exit code
- assert returncode == 1
+def test_exception_at_open_spider_in_pipeline():
+ returncode = _execute("exception")
+ # An exception in pipeline's open_spider should result in a non-zero exit code
+ assert returncode == 1
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 9f81e8602..bb4d96f00 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -89,6 +89,7 @@ class TestCommandSettings:
args=["-s", f"FEEDS={feeds_json}", "spider.py"]
)
self.command.process_options(args, opts)
+ assert self.command.settings is not None
assert isinstance(self.command.settings["FEEDS"], scrapy.settings.BaseSettings)
assert dict(self.command.settings["FEEDS"]) == json.loads(feeds_json)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index be6c80429..8c1716345 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -356,9 +356,10 @@ with multiples lines
@coroutine_test
async def test_engine_status(self, mockserver: MockServer) -> None:
- est = []
+ est: list[list[tuple[str, Any]]] = []
def cb(response):
+ assert crawler.engine
est.append(get_engine_status(crawler.engine))
crawler = get_crawler(SingleRequestSpider)
@@ -373,9 +374,10 @@ with multiples lines
@coroutine_test
async def test_format_engine_status(self, mockserver: MockServer) -> None:
- est = []
+ est: list[str] = []
def cb(response):
+ assert crawler.engine
est.append(format_engine_status(crawler.engine))
crawler = get_crawler(SingleRequestSpider)
@@ -386,8 +388,8 @@ with multiples lines
assert len(est) == 1, est
est = est[0].split("\n")[2:-2] # remove header & footer
# convert to dict
- est = [x.split(":") for x in est]
- est = [x for sublist in est for x in sublist] # flatten
+ est_split = [x.split(":") for x in est]
+ est = [x for sublist in est_split for x in sublist] # flatten
est = [x.lstrip().rstrip() for x in est]
it = iter(est)
s = dict(zip(it, it, strict=True))
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index adac32df1..6a1eea027 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -523,6 +523,7 @@ class TestCrawlerLogging:
}
async def start(self):
+ assert crawler.stats
info_count_start = crawler.stats.get_value("log_count/INFO")
logging.debug("debug message") # noqa: LOG015
logging.info("info message") # noqa: LOG015
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 578e84359..41c7ce526 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import os
import re
from configparser import ConfigParser
@@ -23,6 +25,7 @@ class TestScrapyUtils:
config_parser.read(tox_config_file_path)
pattern = r"Twisted==([\d.]+)"
match = re.search(pattern, config_parser["min"]["deps"])
+ assert match
pinned_twisted_version_string = match[1]
assert twisted_version.short() == pinned_twisted_version_string
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 275199a76..4af5ca0e8 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -16,7 +16,7 @@ from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Request
+from scrapy.http import Request, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.misc import build_from_crawler
@@ -354,6 +354,7 @@ class TestDataURI:
response = await self.download_request(request)
assert response.text == "A brief note"
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
+ assert isinstance(response, TextResponse)
assert response.encoding == "US-ASCII"
@coroutine_test
@@ -362,6 +363,7 @@ class TestDataURI:
response = await self.download_request(request)
assert response.text == "\u038e\u03a3\u038e"
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
+ assert isinstance(response, TextResponse)
assert response.encoding == "iso-8859-7"
@coroutine_test
@@ -370,6 +372,7 @@ class TestDataURI:
response = await self.download_request(request)
assert response.text == "\u038e\u03a3\u038e"
assert response.body == b"\xbe\xd3\xbe"
+ assert isinstance(response, TextResponse)
assert response.encoding == "iso-8859-7"
@coroutine_test
@@ -382,6 +385,7 @@ class TestDataURI:
response = await self.download_request(request)
assert response.text == "\u038e\u03a3\u038e"
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
+ assert isinstance(response, TextResponse)
assert response.encoding == "utf-8"
@coroutine_test
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index d321c9047..92eb18d33 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -233,7 +233,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
return result
def process_request(self, request):
- d = Deferred()
+ d: Deferred[Response] = Deferred()
d.addCallback(self.cb)
d.callback(resp)
return d
@@ -298,6 +298,8 @@ class TestDownloadDeprecated(TestManagerBase):
return succeed(resp)
async with self.get_mwman() as mwman:
+ assert mwman.crawler
+ assert mwman.crawler.spider
with pytest.warns(
ScrapyDeprecationWarning,
match=r"DownloaderMiddlewareManager.download\(\) is deprecated, use download_async\(\) instead",
diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py
index e697de9b7..8c89c3ffb 100644
--- a/tests/test_downloadermiddleware_defaultheaders.py
+++ b/tests/test_downloadermiddleware_defaultheaders.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
@@ -5,28 +7,29 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
-class TestDefaultHeadersMiddleware:
- def get_defaults_mw(self):
- crawler = get_crawler(Spider)
- defaults = {
- to_bytes(k): [to_bytes(v)]
- for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
- }
- return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
-
- def test_process_request(self):
- defaults, mw = self.get_defaults_mw()
- req = Request("http://www.scrapytest.org")
- mw.process_request(req)
- assert req.headers == defaults
-
- def test_update_headers(self):
- defaults, mw = self.get_defaults_mw()
- headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
- bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
- req = Request("http://www.scrapytest.org", headers=headers)
- assert req.headers == bytes_headers
-
- mw.process_request(req)
- defaults.update(bytes_headers)
- assert req.headers == defaults
+def get_defaults_mw() -> tuple[dict[bytes, list[bytes]], DefaultHeadersMiddleware]:
+ crawler = get_crawler(Spider)
+ defaults = {
+ to_bytes(k): [to_bytes(v)]
+ for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
+ }
+ return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
+
+
+def test_process_request():
+ defaults, mw = get_defaults_mw()
+ req = Request("http://www.scrapytest.org")
+ mw.process_request(req)
+ assert req.headers == defaults
+
+
+def test_update_headers():
+ defaults, mw = get_defaults_mw()
+ headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
+ bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
+ req = Request("http://www.scrapytest.org", headers=headers)
+ assert req.headers == bytes_headers
+
+ mw.process_request(req)
+ defaults.update(bytes_headers)
+ assert req.headers == defaults
diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py
index e6b17960e..9b64cf349 100644
--- a/tests/test_downloadermiddleware_downloadtimeout.py
+++ b/tests/test_downloadermiddleware_downloadtimeout.py
@@ -1,43 +1,51 @@
+from __future__ import annotations
+
+from typing import Any
+
from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-class TestDownloadTimeoutMiddleware:
- def get_request_spider_mw(self, settings=None):
- crawler = get_crawler(Spider, settings)
- spider = crawler._create_spider("foo")
- request = Request("http://scrapytest.org/")
- return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
-
- def test_default_download_timeout(self):
- req, spider, mw = self.get_request_spider_mw()
- mw.spider_opened(spider)
- assert mw.process_request(req) is None
- assert req.meta.get("download_timeout") == 180
-
- def test_string_download_timeout(self):
- req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
- mw.spider_opened(spider)
- assert mw.process_request(req) is None
- assert req.meta.get("download_timeout") == 20.1
-
- def test_setting_has_download_timeout(self):
- req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
- mw.spider_opened(spider)
- assert mw.process_request(req) is None
- assert req.meta.get("download_timeout") == 2
-
- def test_request_has_download_timeout(self):
- req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
- mw.spider_opened(spider)
- req.meta["download_timeout"] = 1
- assert mw.process_request(req) is None
- assert req.meta.get("download_timeout") == 1
-
- def test_zero_download_timeout(self):
- req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})
- mw.spider_opened(spider)
- assert mw.process_request(req) is None
- assert req.meta.get("download_timeout") is None
+def get_request_spider_mw(settings: dict[str, Any] | None = None):
+ crawler = get_crawler(Spider, settings)
+ spider = crawler._create_spider("foo")
+ request = Request("http://scrapytest.org/")
+ return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
+
+
+def test_default_download_timeout():
+ req, spider, mw = get_request_spider_mw()
+ mw.spider_opened(spider)
+ assert mw.process_request(req) is None
+ assert req.meta.get("download_timeout") == 180
+
+
+def test_string_download_timeout():
+ req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
+ mw.spider_opened(spider)
+ assert mw.process_request(req) is None
+ assert req.meta.get("download_timeout") == 20.1
+
+
+def test_setting_has_download_timeout():
+ req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
+ mw.spider_opened(spider)
+ assert mw.process_request(req) is None
+ assert req.meta.get("download_timeout") == 2
+
+
+def test_request_has_download_timeout():
+ req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
+ mw.spider_opened(spider)
+ req.meta["download_timeout"] = 1
+ assert mw.process_request(req) is None
+ assert req.meta.get("download_timeout") == 1
+
+
+def test_zero_download_timeout():
+ req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})
+ mw.spider_opened(spider)
+ assert mw.process_request(req) is None
+ assert req.meta.get("download_timeout") is None
diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py
index 539183cd6..0c925a0d1 100644
--- a/tests/test_downloadermiddleware_useragent.py
+++ b/tests/test_downloadermiddleware_useragent.py
@@ -1,33 +1,37 @@
+from __future__ import annotations
+
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-class TestUserAgentMiddleware:
- def get_spider_and_mw(self, default_useragent):
- crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
- spider = crawler._create_spider("foo")
- return spider, UserAgentMiddleware.from_crawler(crawler)
-
- def test_default_agent(self):
- _, mw = self.get_spider_and_mw("default_useragent")
- req = Request("http://scrapytest.org/")
- assert mw.process_request(req) is None
- assert req.headers["User-Agent"] == b"default_useragent"
-
- def test_header_agent(self):
- spider, mw = self.get_spider_and_mw("default_useragent")
- mw.spider_opened(spider)
- req = Request(
- "http://scrapytest.org/", headers={"User-Agent": "header_useragent"}
- )
- assert mw.process_request(req) is None
- assert req.headers["User-Agent"] == b"header_useragent"
-
- def test_no_agent(self):
- spider, mw = self.get_spider_and_mw(None)
- mw.spider_opened(spider)
- req = Request("http://scrapytest.org/")
- assert mw.process_request(req) is None
- assert "User-Agent" not in req.headers
+def get_spider_and_mw(
+ default_useragent: str | None,
+) -> tuple[Spider, UserAgentMiddleware]:
+ crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
+ spider = crawler._create_spider("foo")
+ return spider, UserAgentMiddleware.from_crawler(crawler)
+
+
+def test_default_agent():
+ _, mw = get_spider_and_mw("default_useragent")
+ req = Request("http://scrapytest.org/")
+ assert mw.process_request(req) is None
+ assert req.headers["User-Agent"] == b"default_useragent"
+
+
+def test_header_agent():
+ spider, mw = get_spider_and_mw("default_useragent")
+ mw.spider_opened(spider)
+ req = Request("http://scrapytest.org/", headers={"User-Agent": "header_useragent"})
+ assert mw.process_request(req) is None
+ assert req.headers["User-Agent"] == b"header_useragent"
+
+
+def test_no_agent():
+ spider, mw = get_spider_and_mw(None)
+ mw.spider_opened(spider)
+ req = Request("http://scrapytest.org/")
+ assert mw.process_request(req) is None
+ assert "User-Agent" not in req.headers
diff --git a/tests/test_engine.py b/tests/test_engine.py
index e51eb4664..2b857d35a 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -38,6 +38,8 @@ from tests import get_testdata
from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING:
+ from collections.abc import AsyncIterator
+
from twisted.python.failure import Failure
from scrapy.core.scheduler import Scheduler
@@ -608,8 +610,8 @@ class TestEngineDownload(TestEngineDownloadAsync):
@coroutine_test
async def test_request_scheduled_signal():
class TestScheduler(BaseScheduler):
- def __init__(self):
- self.enqueued = []
+ def __init__(self) -> None:
+ self.enqueued: list[Request] = []
def enqueue_request(self, request: Request) -> bool:
self.enqueued.append(request)
@@ -621,9 +623,9 @@ async def test_request_scheduled_signal():
crawler = get_crawler(MySpider)
engine = ExecutionEngine(crawler, lambda _: None)
- scheduler = TestScheduler()
+ scheduler = TestScheduler() # type: ignore[abstract]
- async def start():
+ async def start() -> AsyncIterator[Any]:
return
yield
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index 18782fb70..0fe2caca4 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -102,7 +102,9 @@ class TestPeriodicLog:
ext.spider_closed(spider, reason="finished")
return ext, a, b
- def check(settings: dict[str, Any], condition: Callable) -> None:
+ def check(
+ settings: dict[str, Any], condition: Callable[[str, Any], bool]
+ ) -> None:
ext, a, b = emulate(settings)
assert list(a["delta"].keys()) == [
k for k, v in ext.stats._stats.items() if condition(k, v)
@@ -168,7 +170,9 @@ class TestPeriodicLog:
ext.spider_closed(spider, reason="finished")
return ext, a, b
- def check(settings: dict[str, Any], condition: Callable) -> None:
+ def check(
+ settings: dict[str, Any], condition: Callable[[str, Any], bool]
+ ) -> None:
ext, a, b = emulate(settings)
assert list(a["stats"].keys()) == [
k for k, v in ext.stats._stats.items() if condition(k, v)
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 0a926479b..469985288 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -378,6 +378,7 @@ class TestBatchDeliveries(TestFeedExportBase):
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(total=2, mockserver=self.mockserver)
+ assert crawler.stats
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12
@@ -392,7 +393,9 @@ class TestBatchDeliveries(TestFeedExportBase):
]
class CustomS3FeedStorage(S3FeedStorage):
- stubs = []
+ from botocore.stub import Stubber # noqa: PLC0415
+
+ stubs: list[Stubber] = []
def open(self, *args, **kwargs):
from botocore import __version__ as botocore_version # noqa: PLC0415
@@ -449,6 +452,7 @@ class TestBatchDeliveries(TestFeedExportBase):
assert len(CustomS3FeedStorage.stubs) == len(items)
for stub in CustomS3FeedStorage.stubs:
stub.assert_no_pending_responses()
+ assert crawler.stats
assert (
"feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats()
)
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index ce9296764..5d4d5a2bd 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
from http.cookiejar import DefaultCookiePolicy
from scrapy.http import Request, Response
@@ -46,7 +48,7 @@ class TestCookieJar:
def test_set_policy(self):
policy = DefaultCookiePolicy()
self.jar.set_policy(policy)
- assert self.jar.jar._policy is policy
+ assert self.jar.jar._policy is policy # type: ignore[attr-defined]
def test_check_expired_frequency(self):
jar = CookieJar(check_expired_frequency=1)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 42ab29267..7c9f717e4 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -1,3 +1,5 @@
+from typing import Any
+
from scrapy.http import (
Headers,
HtmlResponse,
@@ -38,13 +40,13 @@ class TestResponseTypes:
]
for source, cls in mappings:
retcls = responsetypes.from_content_disposition(source)
- assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
def test_from_content_disposition_no_filename(self):
assert responsetypes.from_content_disposition(b"attachment") is Response
def test_from_content_type(self):
- mappings = [
+ mappings: list[tuple[str | bytes, type[Response]]] = [
("text/html; charset=UTF-8", HtmlResponse),
("text/xml; charset=UTF-8", XmlResponse),
("application/xhtml+xml; charset=UTF-8", HtmlResponse),
@@ -58,7 +60,7 @@ class TestResponseTypes:
]
for source, cls in mappings:
retcls = responsetypes.from_content_type(source)
- assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
def test_from_body(self):
mappings = [
@@ -71,7 +73,7 @@ class TestResponseTypes:
]
for source, cls in mappings:
retcls = responsetypes.from_body(source)
- assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+ assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
def test_from_headers(self):
mappings = [
@@ -98,7 +100,7 @@ class TestResponseTypes:
def test_from_args(self):
# TODO: add more tests that check precedence between the different arguments
- mappings = [
+ mappings: list[tuple[dict[str, Any], type[Response]]] = [
({"url": "http://www.example.com/data.csv"}, TextResponse),
# headers takes precedence over url
(
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 8637de41e..39ce87b65 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -379,13 +379,14 @@ class TestIntegrationWithDownloaderAwareInMemory:
url = mockserver.url("/status?n=200", is_secure=False)
start_urls = [url] * 6
yield self.crawler.crawl(start_urls)
+ assert self.crawler.stats
assert self.crawler.stats.get_value("downloader/response_count") == len(
start_urls
)
class TestIncompatibility:
- def _incompatible(self):
+ def _incompatible(self) -> None:
settings = {
"SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue",
"CONCURRENT_REQUESTS_PER_IP": 1,
diff --git a/tests/test_signals.py b/tests/test_signals.py
index f5f13b992..fc9520f49 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -1,13 +1,20 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
import pytest
from scrapy import Request, Spider, signals
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
-from tests.mockserver.http import MockServer
-from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+ from tests.mockserver.http import MockServer
class ItemSpider(Spider):
name = "itemspider"
+ mockserver: MockServer
async def start(self):
for index in range(10):
@@ -34,15 +41,6 @@ class TestMain:
class TestMockServer:
- @classmethod
- def setup_class(cls):
- cls.mockserver = MockServer()
- cls.mockserver.__enter__()
-
- @classmethod
- def teardown_class(cls):
- cls.mockserver.__exit__(None, None, None)
-
def setup_method(self):
self.items = []
@@ -51,11 +49,11 @@ class TestMockServer:
self.items.append(item)
@pytest.mark.only_asyncio
- @inline_callbacks_test
- def test_simple_pipeline(self):
+ @coroutine_test
+ async def test_simple_pipeline(self, mockserver: MockServer) -> None:
crawler = get_crawler(ItemSpider)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
- yield crawler.crawl(mockserver=self.mockserver)
+ await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 10
for index in range(10):
assert {"index": index} in self.items
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index 9257e0232..7cef2fefa 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -1,7 +1,7 @@
from __future__ import annotations
from asyncio import sleep
-from typing import Any
+from typing import TYPE_CHECKING, Any
import pytest
@@ -12,6 +12,9 @@ from scrapy.utils.test import get_crawler
from .utils import twisted_sleep
from .utils.decorators import coroutine_test
+if TYPE_CHECKING:
+ from collections.abc import AsyncIterator, Callable
+
SLEEP_SECONDS = 0.1
ITEM_A = {"id": "a"}
@@ -67,7 +70,11 @@ class TestMain:
await self._test_spider(TestSpider, [ITEM_A])
- async def _test_start(self, start_, expected_items=None):
+ async def _test_start(
+ self,
+ start_: Callable[[Any], AsyncIterator[Any]],
+ expected_items: list[Any] | None = None,
+ ) -> None:
class TestSpider(Spider):
name = "test"
start = start_
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index e5891474b..7824d5c20 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -186,7 +186,7 @@ class ProcessSpiderExceptionSimpleIterableMiddleware:
class ProcessSpiderExceptionAsyncIteratorMiddleware:
async def process_spider_exception(self, response, exception):
yield {"foo": 1}
- d = defer.Deferred()
+ d: defer.Deferred[None] = defer.Deferred()
call_later(0, d.callback, None)
await maybe_deferred_to_future(d)
yield {"foo": 2}
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 6a054ded5..2ba083fd6 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -22,6 +22,7 @@ class _HttpErrorSpider(MockServerSpider):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
+ assert self.mockserver
self.start_urls = [
self.mockserver.url("/status?n=200"),
self.mockserver.url("/status?n=404"),
@@ -36,7 +37,7 @@ class _HttpErrorSpider(MockServerSpider):
for url in self.start_urls:
yield Request(url, self.parse, errback=self.on_error)
- def parse(self, response):
+ def parse(self, response: Response) -> None:
self.parsed.add(response.url[-3:])
def on_error(self, failure):
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 32e83f990..7591c0e7b 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import warnings
-from typing import Any
+from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse
import pytest
@@ -35,6 +35,9 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
+if TYPE_CHECKING:
+ from collections.abc import AsyncIterator
+
class TestRefererMiddleware:
req_meta: dict[str, Any] = {}
@@ -1031,7 +1034,7 @@ async def test_response_policy_only_supports_policy_names():
crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"})
mw = build_from_crawler(RefererMiddleware, crawler)
- async def input_result():
+ async def input_result() -> AsyncIterator[Any]:
yield Request("https://example.com/")
response = Response(
@@ -1079,7 +1082,7 @@ async def test_referer_policies_setting():
)
mw = build_from_crawler(RefererMiddleware, crawler)
- async def input_result():
+ async def input_result() -> AsyncIterator[Any]:
yield Request("https://example.com/")
# "no-referrer-when-downgrade": None,
diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py
index def3a3df3..e879f3a32 100644
--- a/tests/test_spidermiddleware_start.py
+++ b/tests/test_spidermiddleware_start.py
@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.start import StartSpiderMiddleware
from scrapy.spiders import Spider
@@ -5,6 +9,9 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
+if TYPE_CHECKING:
+ from collections.abc import AsyncIterator
+
class TestMiddleware:
@coroutine_test
@@ -12,7 +19,7 @@ class TestMiddleware:
crawler = get_crawler(Spider)
mw = build_from_crawler(StartSpiderMiddleware, crawler)
- async def start():
+ async def start() -> AsyncIterator[Request]:
yield Request("data:,1")
yield Request("data:,2", meta={"is_start_request": True})
yield Request("data:,2", meta={"is_start_request": False})
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index a198d1c09..7528dc51a 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -72,7 +72,7 @@ class TestParallelAsyncio:
for length in [20, 50, 100]:
parallel_count = [0]
max_parallel_count = [0]
- results = []
+ results: list[int] = []
ait = self.get_async_iterable(length)
await _parallel_asyncio(
ait,
@@ -91,7 +91,7 @@ class TestParallelAsyncio:
for length in [20, 50, 100]:
parallel_count = [0]
max_parallel_count = [0]
- results = []
+ results: list[int] = []
ait = self.get_async_iterable_with_delays(length)
await _parallel_asyncio(
ait,
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index b6a9d8b06..518cc3518 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import Any
+
import pytest
from scrapy.exceptions import UsageError
@@ -106,7 +110,7 @@ class TestFeedExportConfig:
)
def test_feed_complete_default_values_from_settings_empty(self):
- feed = {}
+ feed: dict[str, Any] = {}
settings = Settings(
{
"FEED_EXPORT_ENCODING": "custom encoding",
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index dc1d96f66..05b272832 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import pytest
from scrapy.utils.console import get_shell_embed_func
@@ -38,4 +40,5 @@ def test_get_shell_embed_func_bpython():
def test_get_shell_embed_func_ipython():
# default shell should be 'ipython'
shell = get_shell_embed_func()
+ assert shell is not None
assert shell.__name__ == "_embed_ipython_shell"
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index 8b02116b8..9f9e24957 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -1,9 +1,16 @@
+from __future__ import annotations
+
import builtins
from io import StringIO
+from typing import TYPE_CHECKING
from unittest import mock
from scrapy.utils.display import pformat, pprint
+if TYPE_CHECKING:
+ from collections.abc import Mapping, Sequence
+ from types import ModuleType
+
value = {"a": 1}
colorized_strings = {
(
@@ -77,14 +84,19 @@ def test_pformat_no_pygments(isatty):
real_import = builtins.__import__
- def mock_import(name, globals_, locals_, fromlist, level):
+ def mock_import(
+ name: str,
+ globals_: Mapping[str, object] | None = None,
+ locals_: Mapping[str, object] | None = None,
+ fromlist: Sequence[str] | None = (),
+ level: int = 0,
+ ) -> ModuleType:
if "pygments" in name:
raise ImportError
return real_import(name, globals_, locals_, fromlist, level)
- builtins.__import__ = mock_import
- assert pformat(value) == plain_string
- builtins.__import__ = real_import
+ with mock.patch("builtins.__import__", mock_import):
+ assert pformat(value) == plain_string
def test_pprint():
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 1276c1392..a59541edb 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -238,7 +238,7 @@ class TestXmliterBase(ABC):
next(my_iter)
def test_xmliter_objtype_exception(self):
- i = self.xmliter(42, "product")
+ i = self.xmliter(42, "product") # type: ignore[arg-type]
with pytest.raises(TypeError):
next(i)
@@ -353,7 +353,7 @@ class TestLxmlXmliter(TestXmliterBase):
assert node.xpath("f:name/text()").getall() == ["African Coffee Table"]
def test_xmliter_objtype_exception(self):
- i = self.xmliter(42, "product")
+ i = self.xmliter(42, "product") # type: ignore[arg-type]
with pytest.raises(TypeError):
next(i)
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index ee552df64..fe57492b6 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -38,7 +38,7 @@ class TestFailureToExcInfo:
assert exc_info == failure_to_exc_info(failure)
def test_non_failure(self):
- assert failure_to_exc_info("test") is None
+ assert failure_to_exc_info("test") is None # type: ignore[arg-type]
class TestTopLevelFormatter:
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index c4c861404..ab6965ed5 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import os
import sys
from pathlib import Path
@@ -37,7 +39,7 @@ class TestUtilsMisc:
with pytest.raises(NameError):
load_object("scrapy.utils.misc.load_object999")
with pytest.raises(TypeError):
- load_object({})
+ load_object({}) # type: ignore[arg-type]
def test_walk_modules(self):
mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
@@ -114,7 +116,7 @@ class TestUtilsMisc:
args = (True, 100.0)
kwargs = {"key": "val"}
- def _test_with_crawler(mock, crawler):
+ def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
build_from_crawler(mock, crawler, *args, **kwargs)
if hasattr(mock, "from_crawler"):
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 4544cd29e..146cdb802 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -38,7 +38,7 @@ def test_open_in_browser():
resp = Response(url, body=body)
with pytest.raises(TypeError):
- open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg
+ open_in_browser(resp, _openfunc=browser_open) # type: ignore[arg-type]
def test_get_meta_refresh():
@@ -320,4 +320,4 @@ def test_open_in_browser_text_response_uses_txt_extension():
def test_open_in_browser_raises_for_unsupported_response_type():
response = Response("http://www.example.com", body=b"binary")
with pytest.raises(TypeError):
- open_in_browser(response, _openfunc=lambda _: True)
+ open_in_browser(response, _openfunc=lambda _: True) # type: ignore[arg-type]
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 9ac0fd0c0..dc615eca6 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -1,4 +1,7 @@
+from __future__ import annotations
+
import asyncio
+from typing import TYPE_CHECKING
import pytest
from pydispatch import dispatcher
@@ -16,6 +19,9 @@ from scrapy.utils.signal import (
from scrapy.utils.test import get_from_asyncio_queue
from tests.utils.decorators import inline_callbacks_test
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
class TestSendCatchLog:
# whether the function being tested returns exceptions or failures
@@ -24,7 +30,7 @@ class TestSendCatchLog:
@inline_callbacks_test
def test_send_catch_log(self):
test_signal = object()
- handlers_called = set()
+ handlers_called: set[Callable[..., None]] = set()
dispatcher.connect(self.error_handler, signal=test_signal)
dispatcher.connect(self.ok_handler, signal=test_signal)
@@ -74,7 +80,7 @@ class TestSendCatchLogDeferred2(TestSendCatchLogDeferred):
def ok_handler(self, arg, handlers_called):
handlers_called.add(self.ok_handler)
assert arg == "test"
- d = defer.Deferred()
+ d: defer.Deferred[str] = defer.Deferred()
call_later(0, d.callback, "OK")
return d
@@ -108,7 +114,7 @@ class TestSendCatchLogAsync2(TestSendCatchLogAsync):
def ok_handler(self, arg, handlers_called):
handlers_called.add(self.ok_handler)
assert arg == "test"
- d = defer.Deferred()
+ d: defer.Deferred[str] = defer.Deferred()
call_later(0, d.callback, "OK")
return d
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index 05971f423..71f0f76ad 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
from scrapy import Spider
from scrapy.http import Request
from scrapy.item import Item
@@ -17,7 +19,7 @@ def test_iterate_spider_output():
r = Request("http://scrapytest.org")
o = object()
- assert list(iterate_spider_output(i)) == [i]
+ assert list(iterate_spider_output(i)) == [i] # type: ignore[call-overload]
assert list(iterate_spider_output(r)) == [r]
assert list(iterate_spider_output(o)) == [o]
assert list(iterate_spider_output([r, i, o])) == [r, i, o]
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 7a7a264d3..2334c76e9 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -79,9 +79,9 @@ def test_get_oldest():
stale entries in `live_refs`, affecting results unless explicitly cleared.
"""
- def _delete_o1():
+ def _delete_o1() -> None:
"""Delete `o1` and ensure it is actually collected on PyPy."""
- nonlocal o1
+ nonlocal o1 # type: ignore[misc]
del o1
if _IS_PYPY:
@@ -89,7 +89,7 @@ def test_get_oldest():
# still exist until the GC runs, so we force a collection cycle.
garbage_collect()
- def _do_asserts():
+ def _do_asserts() -> None:
assert trackref.get_oldest("Foo") is o1
assert trackref.get_oldest("Bar") is o2
# Ensure the newer Foo is not incorrectly considered the oldest