Skip to content

The agent engineering platform.

standard-tests[minor]: Add standard tests for cache (#23357)

Eugene Yurtsevd90379210a75
diff --git a/libs/standard-tests/langchain_standard_tests/integration_tests/cache.py b/libs/standard-tests/langchain_standard_tests/integration_tests/cache.py
new file mode 100644
index 0000000000..fe84d8450c
--- /dev/null
+++ b/libs/standard-tests/langchain_standard_tests/integration_tests/cache.py
@@ -0,0 +1,192 @@
+from abc import ABC, abstractmethod
+
+import pytest
+from langchain_core.caches import BaseCache
+from langchain_core.outputs import Generation
+
+
+class SyncCacheTestSuite(ABC):
+ """Test suite for checking the BaseCache API of a caching layer for LLMs.
+
+ This test suite verifies the basic caching API of a caching layer for LLMs.
+
+ The test suite is designed for synchronous caching layers.
+
+ Implementers should subclass this test suite and provide a fixture
+ that returns an empty cache for each test.
+ """
+
+ @abstractmethod
+ @pytest.fixture
+ def cache(self) -> BaseCache:
+ """Get the cache class to test.
+
+ The returned cache should be EMPTY.
+ """
+
+ def get_sample_prompt(self) -> str:
+ """Return a sample prompt for testing."""
+ return "Sample prompt for testing."
+
+ def get_sample_llm_string(self) -> str:
+ """Return a sample LLM string for testing."""
+ return "Sample LLM string configuration."
+
+ def get_sample_generation(self) -> Generation:
+ """Return a sample Generation object for testing."""
+ return Generation(
+ text="Sample generated text.", generation_info={"reason": "test"}
+ )
+
+ def test_cache_is_empty(self, cache: BaseCache) -> None:
+ """Test that the cache is empty."""
+ assert (
+ cache.lookup(self.get_sample_prompt(), self.get_sample_llm_string()) is None
+ )
+
+ def test_update_cache(self, cache: BaseCache) -> None:
+ """Test updating the cache."""
+ prompt = self.get_sample_prompt()
+ llm_string = self.get_sample_llm_string()
+ generation = self.get_sample_generation()
+ cache.update(prompt, llm_string, [generation])
+ assert cache.lookup(prompt, llm_string) == [generation]
+
+ def test_cache_still_empty(self, cache: BaseCache) -> None:
+ """This test should follow a test that updates the cache.
+
+ This just verifies that the fixture is set up properly to be empty
+ after each test.
+ """
+ assert (
+ cache.lookup(self.get_sample_prompt(), self.get_sample_llm_string()) is None
+ )
+
+ def test_clear_cache(self, cache: BaseCache) -> None:
+ """Test clearing the cache."""
+ prompt = self.get_sample_prompt()
+ llm_string = self.get_sample_llm_string()
+ generation = self.get_sample_generation()
+ cache.update(prompt, llm_string, [generation])
+ cache.clear()
+ assert cache.lookup(prompt, llm_string) is None
+
+ def test_cache_miss(self, cache: BaseCache) -> None:
+ """Test cache miss."""
+ assert cache.lookup("Nonexistent prompt", self.get_sample_llm_string()) is None
+
+ def test_cache_hit(self, cache: BaseCache) -> None:
+ """Test cache hit."""
+ prompt = self.get_sample_prompt()
+ llm_string = self.get_sample_llm_string()
+ generation = self.get_sample_generation()
+ cache.update(prompt, llm_string, [generation])
+ assert cache.lookup(prompt, llm_string) == [generation]
+
+ def test_update_cache_with_multiple_generations(self, cache: BaseCache) -> None:
+ """Test updating the cache with multiple Generation objects."""
+ prompt = self.get_sample_prompt()
+ llm_string = self.get_sample_llm_string()
+ generations = [
+ self.get_sample_generation(),
+ Generation(text="Another generated text."),
+ ]
+ cache.update(prompt, llm_string, generations)
+ assert cache.lookup(prompt, llm_string) == generations
+
+
+class AsyncCacheTestSuite(ABC):
+ """Test suite for checking the BaseCache API of a caching layer for LLMs.
+
+ This test suite verifies the basic caching API of a caching layer for LLMs.
+
+ The test suite is designed for synchronous caching layers.
+
+ Implementers should subclass this test suite and provide a fixture
+ that returns an empty cache for each test.
+ """
+
+ @abstractmethod
+ @pytest.fixture
+ async def cache(self) -> BaseCache:
+ """Get the cache class to test.
+
+ The returned cache should be EMPTY.
+ """
+
+ def get_sample_prompt(self) -> str:
+ """Return a sample prompt for testing."""
+ return "Sample prompt for testing."
+
+ def get_sample_llm_string(self) -> str:
+ """Return a sample LLM string for testing."""
+ return "Sample LLM string configuration."
+
+ def get_sample_generation(self) -> Generation:
+ """Return a sample Generation object for testing."""
+ return Generation(
+ text="Sample generated text.", generation_info={"reason": "test"}
+ )
+
+ async def test_cache_is_empty(self, cache: BaseCache) -> None:
+ """Test that the cache is empty."""
+ assert (
+ await cache.alookup(self.get_sample_prompt(), self.get_sample_llm_string())
+ is None
+ )
+
+ async def test_update_cache(self, cache: BaseCache) -> None:
+ """Test updating the cache."""
+ prompt = self.get_sample_prompt()
+ llm_string = self.get_sample_llm_string()
+ generation = self.get_sample_generation()
+ await cache.aupdate(prompt, llm_string, [generation])
+ assert await cache.alookup(prompt, llm_string) == [generation]
+
+ async def test_cache_still_empty(self, cache: BaseCache) -> None:
+ """This test should follow a test that updates the cache.
+
+ This just verifies that the fixture is set up properly to be empty
+ after each test.
+ """
+ assert (
+ await cache.alookup(self.get_sample_prompt(), self.get_sample_llm_string())
+ is None
+ )
+
+ async def test_clear_cache(self, cache: BaseCache) -> None:
+ """Test clearing the cache."""
+ prompt = self.get_sample_prompt()
+ llm_string = self.get_sample_llm_string()
+ generation = self.get_sample_generation()
+ await cache.aupdate(prompt, llm_string, [generation])
+ await cache.aclear()
+ assert await cache.alookup(prompt, llm_string) is None
+
+ async def test_cache_miss(self, cache: BaseCache) -> None:
+ """Test cache miss."""
+ assert (
+ await cache.alookup("Nonexistent prompt", self.get_sample_llm_string())
+ is None
+ )
+
+ async def test_cache_hit(self, cache: BaseCache) -> None:
+ """Test cache hit."""
+ prompt = self.get_sample_prompt()
+ llm_string = self.get_sample_llm_string()
+ generation = self.get_sample_generation()
+ await cache.aupdate(prompt, llm_string, [generation])
+ assert await cache.alookup(prompt, llm_string) == [generation]
+
+ async def test_update_cache_with_multiple_generations(
+ self, cache: BaseCache
+ ) -> None:
+ """Test updating the cache with multiple Generation objects."""
+ prompt = self.get_sample_prompt()
+ llm_string = self.get_sample_llm_string()
+ generations = [
+ self.get_sample_generation(),
+ Generation(text="Another generated text."),
+ ]
+ await cache.aupdate(prompt, llm_string, generations)
+ assert await cache.alookup(prompt, llm_string) == generations
diff --git a/libs/standard-tests/poetry.lock b/libs/standard-tests/poetry.lock
index 91e3029387..f0c71bc886 100644
--- a/libs/standard-tests/poetry.lock
+++ b/libs/standard-tests/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
[[package]]
name = "annotated-types"
@@ -605,6 +605,24 @@ tomli = {version = ">=1", markers = "python_version < \"3.11\""}
[package.extras]
dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+[[package]]
+name = "pytest-asyncio"
+version = "0.23.7"
+description = "Pytest support for asyncio"
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "pytest_asyncio-0.23.7-py3-none-any.whl", hash = "sha256:009b48127fbe44518a547bddd25611551b0e43ccdbf1e67d12479f569832c20b"},
+ {file = "pytest_asyncio-0.23.7.tar.gz", hash = "sha256:5f5c72948f4c49e7db4f29f2521d4031f1c27f86e57b046126654083d4770268"},
+]
+
+[package.dependencies]
+pytest = ">=7.0.0,<9"
+
+[package.extras]
+docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"]
+testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"]
+
[[package]]
name = "pyyaml"
version = "6.0.1"
@@ -630,7 +648,6 @@ files = [
{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
- {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -780,4 +797,4 @@ zstd = ["zstandard (>=0.18.0)"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.8.1,<4.0"
-content-hash = "a27025e6afa0275f153a9fd98c890c16187f9d01f5ca0b60aae23cee1a7d9dcc"
+content-hash = "fe8e04975482a0f8e67d07d186c401de1d321068dd1e595c836b156f7c4fcd9c"
diff --git a/libs/standard-tests/pyproject.toml b/libs/standard-tests/pyproject.toml
index d4d3b4472a..7b8a230f61 100644
--- a/libs/standard-tests/pyproject.toml
+++ b/libs/standard-tests/pyproject.toml
@@ -21,6 +21,7 @@ optional = true
[tool.poetry.group.test.dependencies]
langchain-core = { path = "../core", develop = true }
+pytest-asyncio = "^0.23.7"
[tool.poetry.group.test_integration]
optional = true
@@ -60,3 +61,24 @@ omit = ["tests/*"]
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+# --strict-markers will raise errors on unknown marks.
+# https://docs.pytest.org/en/7.1.x/how-to/mark.html#raising-errors-on-unknown-marks
+#
+# https://docs.pytest.org/en/7.1.x/reference/reference.html
+# --strict-config any warnings encountered while parsing the `pytest`
+# section of the configuration file raise errors.
+#
+# https://github.com/tophat/syrupy
+# --snapshot-warn-unused Prints a warning on unused snapshots rather than fail the test suite.
+addopts = "--strict-markers --strict-config --durations=5 -vv"
+# Registering custom markers.
+# https://docs.pytest.org/en/7.1.x/example/markers.html#registering-markers
+markers = [
+ "requires: mark tests as requiring a specific library",
+ "scheduled: mark tests to run in scheduled testing",
+ "compile: mark placeholder test used to compile integration tests without running them",
+]
+asyncio_mode = "auto"
+
diff --git a/libs/standard-tests/tests/unit_tests/test_in_memory_cache.py b/libs/standard-tests/tests/unit_tests/test_in_memory_cache.py
new file mode 100644
index 0000000000..4f67a87649
--- /dev/null
+++ b/libs/standard-tests/tests/unit_tests/test_in_memory_cache.py
@@ -0,0 +1,19 @@
+import pytest
+from langchain_core.caches import InMemoryCache
+
+from langchain_standard_tests.integration_tests.cache import (
+ AsyncCacheTestSuite,
+ SyncCacheTestSuite,
+)
+
+
+class TestInMemoryCache(SyncCacheTestSuite):
+ @pytest.fixture
+ def cache(self) -> InMemoryCache:
+ return InMemoryCache()
+
+
+class TestInMemoryCacheAsync(AsyncCacheTestSuite):
+ @pytest.fixture
+ async def cache(self) -> InMemoryCache:
+ return InMemoryCache()