The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
[Feature] Add support for vllm as llm source (#1149)
Deshraj Yadav0373fa231caa
diff --git a/configs/vllm.yaml b/configs/vllm.yamlnew file mode 100644index 000000000..536a589a1--- /dev/null+++ b/configs/vllm.yaml@@ -0,0 +1,14 @@+llm:+ provider: vllm+ config:+ model: 'meta-llama/Llama-2-70b-hf'+ temperature: 0.5+ top_p: 1+ top_k: 10+ stream: true+ trust_remote_code: true++embedder:+ provider: huggingface+ config:+ model: 'BAAI/bge-small-en-v1.5'diff --git a/docs/components/llms.mdx b/docs/components/llms.mdxindex faf64855b..3825d0175 100644--- a/docs/components/llms.mdx+++ b/docs/components/llms.mdx@@ -14,6 +14,7 @@ Embedchain comes with built-in support for various popular large language models<Card title="Cohere" href="#cohere"></Card><Card title="Together" href="#together"></Card><Card title="Ollama" href="#ollama"></Card>+ <Card title="vLLM" href="#vllm"></Card><Card title="GPT4All" href="#gpt4all"></Card><Card title="JinaChat" href="#jinachat"></Card><Card title="Hugging Face" href="#hugging-face"></Card>@@ -393,6 +394,34 @@ llm:</CodeGroup>+## Ollama++Setup vLLM by following instructions given in [their docs](https://docs.vllm.ai/en/latest/getting_started/installation.html).++<CodeGroup>++```python main.py+import os+from embedchain import App++# load llm configuration from config.yaml file+app = App.from_config(config_path="config.yaml")+```++```yaml config.yaml+llm:+ provider: vllm+ config:+ model: 'meta-llama/Llama-2-70b-hf'+ temperature: 0.5+ top_p: 1+ top_k: 10+ stream: true+ trust_remote_code: true+```++</CodeGroup>+## GPT4ALLInstall related dependencies using the following command:@@ -515,7 +544,7 @@ app = App.from_config(config_path="config.yaml")```yaml config.yamlllm:- provider: huggingface+ provider: huggingfaceconfig:endpoint: https://api-inference.huggingface.co/models/gpt2 # replace with your personal endpoint```@@ -525,7 +554,7 @@ If your endpoint requires additional parameters, you can pass them in the `model```llm:- provider: huggingface+ provider: huggingfaceconfig:endpoint: <YOUR_ENDPOINT_URL_HERE>model_kwargs:diff --git a/embedchain/app.py b/embedchain/app.pyindex ab3643d38..45f422c98 100644--- a/embedchain/app.py+++ b/embedchain/app.py@@ -9,14 +9,9 @@ from typing import Any, Dict, Optionalimport requestsimport yaml-from embedchain.cache import (- Config,- ExactMatchEvaluation,- SearchDistanceEvaluation,- cache,- gptcache_data_manager,- gptcache_pre_function,-)+from embedchain.cache import (Config, ExactMatchEvaluation,+ SearchDistanceEvaluation, cache,+ gptcache_data_manager, gptcache_pre_function)from embedchain.client import Clientfrom embedchain.config import AppConfig, CacheConfig, ChunkerConfigfrom embedchain.constants import SQLITE_PATHdiff --git a/embedchain/config/llm/base.py b/embedchain/config/llm/base.pyindex 4748aa6c7..7fa84c94c 100644--- a/embedchain/config/llm/base.py+++ b/embedchain/config/llm/base.py@@ -73,7 +73,7 @@ class BaseLlmConfig(BaseConfig):callbacks: Optional[List] = None,api_key: Optional[str] = None,endpoint: Optional[str] = None,- model_kwargs: Optional[Dict[str, Any]] = {},+ model_kwargs: Optional[Dict[str, Any]] = None,):"""Initializes a configuration class instance for the LLM.@@ -115,6 +115,8 @@ class BaseLlmConfig(BaseConfig)::type model_kwargs: Optional[Dict[str, Any]], optional:param callbacks: Langchain callback functions to use, defaults to None:type callbacks: Optional[List], optional+ :param query_type: The type of query to use, defaults to None+ :type query_type: Optional[str], optional:raises ValueError: If the template is not valid as template shouldcontain $context and $query (and optionally $history):raises ValueError: Stream is not boolean@@ -142,6 +144,7 @@ class BaseLlmConfig(BaseConfig):self.api_key = api_keyself.endpoint = endpointself.model_kwargs = model_kwargs+if type(prompt) is str:prompt = Template(prompt)diff --git a/embedchain/embedchain.py b/embedchain/embedchain.pyindex 3e5babffc..148d1f883 100644--- a/embedchain/embedchain.py+++ b/embedchain/embedchain.py@@ -7,7 +7,12 @@ from typing import Any, Dict, List, Optional, Tuple, Unionfrom dotenv import load_dotenvfrom langchain.docstore.document import Document-from embedchain.cache import adapt, get_gptcache_session, gptcache_data_convert, gptcache_update_cache_callback+from embedchain.cache import (+ adapt,+ get_gptcache_session,+ gptcache_data_convert,+ gptcache_update_cache_callback,+)from embedchain.chunkers.base_chunker import BaseChunkerfrom embedchain.config import AddConfig, BaseLlmConfig, ChunkerConfigfrom embedchain.config.base_app_config import BaseAppConfigdiff --git a/embedchain/llm/base.py b/embedchain/llm/base.pyindex 4cf138712..ba464da2c 100644--- a/embedchain/llm/base.py+++ b/embedchain/llm/base.py@@ -4,7 +4,9 @@ from typing import Any, Dict, Generator, List, Optionalfrom langchain.schema import BaseMessage as LCBaseMessagefrom embedchain.config import BaseLlmConfig-from embedchain.config.llm.base import DEFAULT_PROMPT, DEFAULT_PROMPT_WITH_HISTORY_TEMPLATE, DOCS_SITE_PROMPT_TEMPLATE+from embedchain.config.llm.base import (DEFAULT_PROMPT,+ DEFAULT_PROMPT_WITH_HISTORY_TEMPLATE,+ DOCS_SITE_PROMPT_TEMPLATE)from embedchain.helpers.json_serializable import JSONSerializablefrom embedchain.memory.base import ChatHistoryfrom embedchain.memory.message import ChatMessagediff --git a/embedchain/llm/vllm.py b/embedchain/llm/vllm.pynew file mode 100644index 000000000..faac1f39d--- /dev/null+++ b/embedchain/llm/vllm.py@@ -0,0 +1,40 @@+from typing import Iterable, Optional, Union++from langchain.callbacks.manager import CallbackManager+from langchain.callbacks.stdout import StdOutCallbackHandler+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler+from langchain_community.llms import VLLM as BaseVLLM++from embedchain.config import BaseLlmConfig+from embedchain.helpers.json_serializable import register_deserializable+from embedchain.llm.base import BaseLlm+++@register_deserializable+class VLLM(BaseLlm):+ def __init__(self, config: Optional[BaseLlmConfig] = None):+ super().__init__(config=config)+ if self.config.model is None:+ self.config.model = "mosaicml/mpt-7b"++ def get_llm_model_answer(self, prompt):+ return self._get_answer(prompt=prompt, config=self.config)++ @staticmethod+ def _get_answer(prompt: str, config: BaseLlmConfig) -> Union[str, Iterable]:+ callback_manager = [StreamingStdOutCallbackHandler()] if config.stream else [StdOutCallbackHandler()]++ # Prepare the arguments for BaseVLLM+ llm_args = {+ "model": config.model,+ "temperature": config.temperature,+ "top_p": config.top_p,+ "callback_manager": CallbackManager(callback_manager),+ }++ # Add model_kwargs if they are not None+ if config.model_kwargs is not None:+ llm_args.update(config.model_kwargs)++ llm = BaseVLLM(**llm_args)+ return llm(prompt)diff --git a/embedchain/vectordb/zilliz.py b/embedchain/vectordb/zilliz.pyindex d49eabf84..a310a0ae4 100644--- a/embedchain/vectordb/zilliz.py+++ b/embedchain/vectordb/zilliz.py@@ -6,7 +6,15 @@ from embedchain.helpers.json_serializable import register_deserializablefrom embedchain.vectordb.base import BaseVectorDBtry:- from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, MilvusClient, connections, utility+ from pymilvus import (+ Collection,+ CollectionSchema,+ DataType,+ FieldSchema,+ MilvusClient,+ connections,+ utility,+ )except ImportError:raise ImportError("Zilliz requires extra dependencies. Install with `pip install --upgrade embedchain[milvus]`"diff --git a/pyproject.toml b/pyproject.tomlindex b2d0ab1a5..bb958e148 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "embedchain"-version = "0.1.57"+version = "0.1.58"description = "Data platform for LLMs - Load, index, retrieve and sync any unstructured data"authors = ["Taranjeet Singh <[email protected]>",