The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
[Improvement] add vector_dimension configuration in embedder config (#1192)
Deven Patel59600e2a5bb0
diff --git a/docs/api-reference/advanced/configuration.mdx b/docs/api-reference/advanced/configuration.mdxindex f16488652..365a223e6 100644--- a/docs/api-reference/advanced/configuration.mdx+++ b/docs/api-reference/advanced/configuration.mdx@@ -8,7 +8,7 @@ You can configure different components of your app (`llm`, `embedding model`, or<Tip>-Embedchain applications are configurable using YAML file, JSON file or by directly passing the config dictionary. Checkout the [docs here](/api-reference/pipeline/overview#usage) on how to use other formats.+Embedchain applications are configurable using YAML file, JSON file or by directly passing the config dictionary. Checkout the [docs here](/api-reference/app/overview#usage) on how to use other formats.</Tip><CodeGroup>@@ -214,7 +214,11 @@ Alright, let's dive into what each key means in the yaml config above:- `provider` (String): The provider for the embedder, set to 'openai'. You can find the full list of embedding model providers in [our docs](/components/embedding-models).- `config`:- `model` (String): The specific model used for text embedding, 'text-embedding-ada-002'.+ - `vector_dimension` (Integer): The vector dimension of the embedding model. [Defaults](https://github.com/embedchain/embedchain/blob/e572b5a3dc1b66f1e9b3357d11a88c63b5ce06e3/embedchain/models/vector_dimensions.py)- `api_key` (String): The API key for the embedding model.+ - `deployment_name` (String): The deployment name for the embedding model.+ - `title` (String): The title for the embedding model for Google Embedder.+ - `task_type` (String): The task type for the embedding model for Google Embedder.5. `chunker` Section:- `chunk_size` (Integer): The size of each chunk of text that is sent to the language model.- `chunk_overlap` (Integer): The amount of overlap between each chunk of text.diff --git a/docs/components/llms.mdx b/docs/components/llms.mdxindex 1a58d7275..bbf05a365 100644--- a/docs/components/llms.mdx+++ b/docs/components/llms.mdx@@ -250,7 +250,7 @@ app = App.from_config(config_path="config.yaml")llm:provider: azure_openaiconfig:- model: gpt-35-turbo+ model: gpt-3.5-turbodeployment_name: your_llm_deployment_nametemperature: 0.5max_tokens: 1000diff --git a/embedchain/config/embedder/base.py b/embedchain/config/embedder/base.pyindex 78bb8b305..dc14cea26 100644--- a/embedchain/config/embedder/base.py+++ b/embedchain/config/embedder/base.py@@ -6,7 +6,11 @@ from embedchain.helpers.json_serializable import register_deserializable@register_deserializableclass BaseEmbedderConfig:def __init__(- self, model: Optional[str] = None, deployment_name: Optional[str] = None, api_key: Optional[str] = None+ self,+ model: Optional[str] = None,+ deployment_name: Optional[str] = None,+ vector_dimension: Optional[int] = None,+ api_key: Optional[str] = None,):"""Initialize a new instance of an embedder config class.@@ -18,4 +22,5 @@ class BaseEmbedderConfig:"""self.model = modelself.deployment_name = deployment_name+ self.vector_dimension = vector_dimensionself.api_key = api_keydiff --git a/embedchain/embedder/google.py b/embedchain/embedder/google.pyindex e69d3b142..4d09f6be6 100644--- a/embedchain/embedder/google.py+++ b/embedchain/embedder/google.py@@ -27,5 +27,5 @@ class GoogleAIEmbedder(BaseEmbedder):embedding_fn = GoogleAIEmbeddingFunction(config=config)self.set_embedding_fn(embedding_fn=embedding_fn)- vector_dimension = VectorDimensions.GOOGLE_AI.value+ vector_dimension = self.config.vector_dimension or VectorDimensions.GOOGLE_AI.valueself.set_vector_dimension(vector_dimension=vector_dimension)diff --git a/embedchain/embedder/gpt4all.py b/embedchain/embedder/gpt4all.pyindex d078825d5..9e17131ae 100644--- a/embedchain/embedder/gpt4all.py+++ b/embedchain/embedder/gpt4all.py@@ -16,5 +16,5 @@ class GPT4AllEmbedder(BaseEmbedder):embedding_fn = BaseEmbedder._langchain_default_concept(embeddings)self.set_embedding_fn(embedding_fn=embedding_fn)- vector_dimension = VectorDimensions.GPT4ALL.value+ vector_dimension = self.config.vector_dimension or VectorDimensions.GPT4ALL.valueself.set_vector_dimension(vector_dimension=vector_dimension)diff --git a/embedchain/embedder/huggingface.py b/embedchain/embedder/huggingface.pyindex 32cae44ef..83433c05f 100644--- a/embedchain/embedder/huggingface.py+++ b/embedchain/embedder/huggingface.py@@ -15,5 +15,5 @@ class HuggingFaceEmbedder(BaseEmbedder):embedding_fn = BaseEmbedder._langchain_default_concept(embeddings)self.set_embedding_fn(embedding_fn=embedding_fn)- vector_dimension = VectorDimensions.HUGGING_FACE.value+ vector_dimension = self.config.vector_dimension or VectorDimensions.HUGGING_FACE.valueself.set_vector_dimension(vector_dimension=vector_dimension)diff --git a/embedchain/embedder/openai.py b/embedchain/embedder/openai.pyindex e71f9c047..5f3114a31 100644--- a/embedchain/embedder/openai.py+++ b/embedchain/embedder/openai.py@@ -32,4 +32,5 @@ class OpenAIEmbedder(BaseEmbedder):model_name=self.config.model,)self.set_embedding_fn(embedding_fn=embedding_fn)- self.set_vector_dimension(vector_dimension=VectorDimensions.OPENAI.value)+ vector_dimension = self.config.vector_dimension or VectorDimensions.OPENAI.value+ self.set_vector_dimension(vector_dimension=vector_dimension)diff --git a/embedchain/embedder/vertexai.py b/embedchain/embedder/vertexai.pyindex fe9476464..c8eed7d93 100644--- a/embedchain/embedder/vertexai.py+++ b/embedchain/embedder/vertexai.py@@ -15,5 +15,5 @@ class VertexAIEmbedder(BaseEmbedder):embedding_fn = BaseEmbedder._langchain_default_concept(embeddings)self.set_embedding_fn(embedding_fn=embedding_fn)- vector_dimension = VectorDimensions.VERTEX_AI.value+ vector_dimension = self.config.vector_dimension or VectorDimensions.VERTEX_AI.valueself.set_vector_dimension(vector_dimension=vector_dimension)diff --git a/embedchain/utils/misc.py b/embedchain/utils/misc.pyindex 0f117b32e..c08665c2f 100644--- a/embedchain/utils/misc.py+++ b/embedchain/utils/misc.py@@ -438,6 +438,7 @@ def validate_config(config_data):Optional("api_key"): str,Optional("title"): str,Optional("task_type"): str,+ Optional("vector_dimension"): int,},},Optional("embedding_model"): {@@ -448,6 +449,7 @@ def validate_config(config_data):Optional("api_key"): str,Optional("title"): str,Optional("task_type"): str,+ Optional("vector_dimension"): int,},},Optional("chunker"): {