The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
community: Memcached LLM Cache Integration (#27323)
Dmitriy Prokopchuk53b0a99f3769
diff --git a/docs/docs/integrations/llm_caching.ipynb b/docs/docs/integrations/llm_caching.ipynbindex ee5152e023..4ba1901613 100644--- a/docs/docs/integrations/llm_caching.ipynb+++ b/docs/docs/integrations/llm_caching.ipynb@@ -2368,6 +2368,102 @@")"]},+ {+ "cell_type": "markdown",+ "id": "7e6b9b1a",+ "metadata": {},+ "source": [+ "## `Memcached` Cache\n",+ "You can use [Memcached](https://www.memcached.org/) as a cache to cache prompts and responses through [pymemcache](https://github.com/pinterest/pymemcache).\n",+ "\n",+ "This cache requires the pymemcache dependency to be installed:"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "id": "b2e5e0b1",+ "metadata": {},+ "outputs": [],+ "source": [+ "%pip install -qU pymemcache"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "id": "4c7ffe37",+ "metadata": {},+ "outputs": [],+ "source": [+ "from langchain_community.cache import MemcachedCache\n",+ "from pymemcache.client.base import Client\n",+ "\n",+ "set_llm_cache(MemcachedCache(Client(\"localhost\")))"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "id": "a4cfc48a",+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CPU times: user 32.8 ms, sys: 21 ms, total: 53.8 ms\n",+ "Wall time: 343 ms\n"+ ]+ },+ {+ "data": {+ "text/plain": [+ "'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side!'"+ ]+ },+ "execution_count": 5,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "%%time\n",+ "# The first time, it is not yet in cache, so it should take longer\n",+ "llm.invoke(\"Tell me a joke\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "id": "cb3b2bf5",+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "CPU times: user 2.31 ms, sys: 850 µs, total: 3.16 ms\n",+ "Wall time: 6.43 ms\n"+ ]+ },+ {+ "data": {+ "text/plain": [+ "'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side!'"+ ]+ },+ "execution_count": 7,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "%%time\n",+ "# The second time it is, so it goes faster\n",+ "llm.invoke(\"Tell me a joke\")"+ ]+ },{"cell_type": "markdown","id": "7019c991-0101-4f9c-b212-5729a5471293",diff --git a/docs/docs/integrations/providers/memcached.mdx b/docs/docs/integrations/providers/memcached.mdxnew file mode 100644index 0000000000..f7719deda4--- /dev/null+++ b/docs/docs/integrations/providers/memcached.mdx@@ -0,0 +1,34 @@+# Memcached++> [Memcached](https://www.memcached.org/) is a free & open source, high-performance, distributed memory object caching system,+> generic in nature, but intended for use in speeding up dynamic web applications by alleviating database load.++This page covers how to use Memcached with langchain, using [pymemcache](https://github.com/pinterest/pymemcache) as+a client to connect to an already running Memcached instance.++## Installation and Setup+```bash+pip install pymemcache+```++## LLM Cache++To integrate a Memcached Cache into your application:+```python3+from langchain.globals import set_llm_cache+from langchain_openai import OpenAI++from langchain_community.cache import MemcachedCache+from pymemcache.client.base import Client++llm = OpenAI(model="gpt-3.5-turbo-instruct", n=2, best_of=2)+set_llm_cache(MemcachedCache(Client('localhost')))++# The first time, it is not yet in cache, so it should take longer+llm.invoke("Which city is the most crowded city in the USA?")++# The second time it is, so it goes faster+llm.invoke("Which city is the most crowded city in the USA?")+```++Learn more in the [example notebook](/docs/integrations/llm_caching#memcached-cache)\ No newline at end of filediff --git a/libs/community/langchain_community/cache.py b/libs/community/langchain_community/cache.pyindex c074747370..697c26ed87 100644--- a/libs/community/langchain_community/cache.py+++ b/libs/community/langchain_community/cache.py@@ -91,6 +91,7 @@ logger = logging.getLogger(__file__)if TYPE_CHECKING:import momento+ import pymemcachefrom astrapy.db import AstraDB, AsyncAstraDBfrom cassandra.cluster import Session as CassandraSession@@ -2599,3 +2600,96 @@ class SingleStoreDBSemanticCache(BaseCache):if index_name in self._cache_dict:self._cache_dict[index_name].drop()del self._cache_dict[index_name]+++class MemcachedCache(BaseCache):+ """Cache that uses Memcached backend through pymemcache client lib"""++ def __init__(self, client_: Any):+ """+ Initialize an instance of MemcachedCache.++ Args:+ client_ (str): An instance of any of pymemcache's Clients+ (Client, PooledClient, HashClient)+ Example:+ .. code-block:: python+ ifrom langchain.globals import set_llm_cache+ from langchain_openai import OpenAI++ from langchain_community.cache import MemcachedCache+ from pymemcache.client.base import Client++ llm = OpenAI(model="gpt-3.5-turbo-instruct", n=2, best_of=2)+ set_llm_cache(MemcachedCache(Client('localhost')))++ # The first time, it is not yet in cache, so it should take longer+ llm.invoke("Which city is the most crowded city in the USA?")++ # The second time it is, so it goes faster+ llm.invoke("Which city is the most crowded city in the USA?")+ """++ try:+ from pymemcache.client import (+ Client,+ HashClient,+ PooledClient,+ RetryingClient,+ )+ except (ImportError, ModuleNotFoundError):+ raise ImportError(+ "Could not import pymemcache python package. "+ "Please install it with `pip install -U pymemcache`."+ )++ if not (+ isinstance(client_, Client)+ or isinstance(client_, PooledClient)+ or isinstance(client_, HashClient)+ or isinstance(client_, RetryingClient)+ ):+ raise ValueError("Please pass a valid pymemcached client")++ self.client = client_++ def lookup(self, prompt: str, llm_string: str) -> Optional[RETURN_VAL_TYPE]:+ """Look up based on prompt and llm_string."""+ key = _hash(prompt + llm_string)+ try:+ result = self.client.get(key)+ except pymemcache.MemcacheError:+ return None++ return _loads_generations(result) if result is not None else None++ def update(self, prompt: str, llm_string: str, return_val: RETURN_VAL_TYPE) -> None:+ """Update cache based on prompt and llm_string."""+ key = _hash(prompt + llm_string)++ # Validate input is made of standard LLM generations+ for gen in return_val:+ if not isinstance(gen, Generation):+ raise ValueError(+ "Memcached only supports caching of normal LLM generations, "+ + f"got {type(gen)}"+ )++ # Deserialize return_val into string and update cache+ value = _dumps_generations(return_val)+ self.client.set(key, value)++ def clear(self, **kwargs: Any) -> None:+ """+ Clear the entire cache. Takes optional kwargs:++ delay: optional int, the number of seconds to wait before flushing,+ or zero to flush immediately (the default). NON-BLOCKING, returns+ immediately.+ noreply: optional bool, True to not wait for the reply (defaults to+ client.default_noreply).+ """+ delay = kwargs.get("delay", 0)+ noreply = kwargs.get("noreply", None)++ self.client.flush_all(delay, noreply)diff --git a/libs/community/tests/integration_tests/cache/test_memcached_cache.py b/libs/community/tests/integration_tests/cache/test_memcached_cache.pynew file mode 100644index 0000000000..2aca3df056--- /dev/null+++ b/libs/community/tests/integration_tests/cache/test_memcached_cache.py@@ -0,0 +1,61 @@+"""+Test Memcached llm cache functionality. Requires running instance of Memcached on+localhost default port (11211) and pymemcache+"""++import pytest+from langchain.globals import get_llm_cache, set_llm_cache+from langchain_core.outputs import Generation, LLMResult++from langchain_community.cache import MemcachedCache+from tests.unit_tests.llms.fake_llm import FakeLLM++DEFAULT_MEMCACHED_URL = "localhost"++[email protected]("pymemcache")+def test_memcached_cache() -> None:+ """Test general Memcached caching"""+ from pymemcache import Client++ set_llm_cache(MemcachedCache(Client(DEFAULT_MEMCACHED_URL)))+ llm = FakeLLM()++ params = llm.dict()+ params["stop"] = None+ llm_string = str(sorted([(k, v) for k, v in params.items()]))+ get_llm_cache().update("foo", llm_string, [Generation(text="fizz")])+ output = llm.generate(["foo"])+ expected_output = LLMResult(+ generations=[[Generation(text="fizz")]],+ llm_output={},+ )+ assert output == expected_output+ # clear the cache+ get_llm_cache().clear()++[email protected]("pymemcache")+def test_memcached_cache_flush() -> None:+ """Test flushing Memcached cache"""+ from pymemcache import Client++ set_llm_cache(MemcachedCache(Client(DEFAULT_MEMCACHED_URL)))+ llm = FakeLLM()++ params = llm.dict()+ params["stop"] = None+ llm_string = str(sorted([(k, v) for k, v in params.items()]))+ get_llm_cache().update("foo", llm_string, [Generation(text="fizz")])+ output = llm.generate(["foo"])+ expected_output = LLMResult(+ generations=[[Generation(text="fizz")]],+ llm_output={},+ )+ assert output == expected_output+ # clear the cache+ get_llm_cache().clear(delay=0, noreply=False)++ # After cache has been cleared, the result shouldn't be the same+ output = llm.generate(["foo"])+ assert output != expected_output