The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
Introduce chunker config in yaml config (#907)
Sidharth Mohantya1de23871633
diff --git a/embedchain/apps/app.py b/embedchain/apps/app.pyindex 8bb6e5bf5..1508027b5 100644--- a/embedchain/apps/app.py+++ b/embedchain/apps/app.py@@ -2,7 +2,7 @@ from typing import Optionalimport yaml-from embedchain.config import AppConfig, BaseEmbedderConfig, BaseLlmConfig+from embedchain.config import AppConfig, BaseEmbedderConfig, BaseLlmConfig, ChunkerConfigfrom embedchain.config.vectordb.base import BaseVectorDbConfigfrom embedchain.embedchain import EmbedChainfrom embedchain.embedder.base import BaseEmbedder@@ -38,6 +38,7 @@ class App(EmbedChain):embedder: BaseEmbedder = None,embedder_config: Optional[BaseEmbedderConfig] = None,system_prompt: Optional[str] = None,+ chunker: Optional[ChunkerConfig] = None,):"""Initialize a new `App` instance.@@ -97,6 +98,9 @@ class App(EmbedChain):if embedder is None:embedder = OpenAIEmbedder(config=embedder_config)+ self.chunker = None+ if chunker:+ self.chunker = ChunkerConfig(**chunker)# Type check assignmentsif not isinstance(llm, BaseLlm):raise TypeError(@@ -137,6 +141,7 @@ class App(EmbedChain):llm_config_data = config_data.get("llm", {})db_config_data = config_data.get("vectordb", {})embedding_model_config_data = config_data.get("embedding_model", config_data.get("embedder", {}))+ chunker_config_data = config_data.get("chunker", {})app_config = AppConfig(**app_config_data.get("config", {}))@@ -148,4 +153,4 @@ class App(EmbedChain):embedder_provider = embedding_model_config_data.get("provider", "openai")embedder = EmbedderFactory.create(embedder_provider, embedding_model_config_data.get("config", {}))- return cls(config=app_config, llm=llm, db=db, embedder=embedder)+ return cls(config=app_config, llm=llm, db=db, embedder=embedder, chunker=chunker_config_data)diff --git a/embedchain/config/add_config.py b/embedchain/config/add_config.pyindex 945ec09ba..16e99d687 100644--- a/embedchain/config/add_config.py+++ b/embedchain/config/add_config.py@@ -1,3 +1,5 @@+import builtins+from importlib import import_modulefrom typing import Callable, Optionalfrom embedchain.config.base_config import BaseConfig@@ -18,7 +20,18 @@ class ChunkerConfig(BaseConfig):):self.chunk_size = chunk_size if chunk_size else 2000self.chunk_overlap = chunk_overlap if chunk_overlap else 0- self.length_function = length_function if length_function else len+ if isinstance(length_function, str):+ self.length_function = self.load_func(length_function)+ else:+ self.length_function = length_function if length_function else len++ def load_func(self, dotpath: str):+ if "." not in dotpath:+ return getattr(builtins, dotpath)+ else:+ module_, func = dotpath.rsplit(".", maxsplit=1)+ m = import_module(module_)+ return getattr(m, func)@register_deserializablediff --git a/embedchain/embedchain.py b/embedchain/embedchain.pyindex 0577bc3bc..eb0dc62a5 100644--- a/embedchain/embedchain.py+++ b/embedchain/embedchain.py@@ -10,15 +10,14 @@ from dotenv import load_dotenvfrom langchain.docstore.document import Documentfrom embedchain.chunkers.base_chunker import BaseChunker-from embedchain.config import AddConfig, BaseLlmConfig+from embedchain.config import AddConfig, BaseLlmConfig, ChunkerConfigfrom embedchain.config.apps.base_app_config import BaseAppConfigfrom embedchain.data_formatter import DataFormatterfrom embedchain.embedder.base import BaseEmbedderfrom embedchain.helper.json_serializable import JSONSerializablefrom embedchain.llm.base import BaseLlmfrom embedchain.loaders.base_loader import BaseLoader-from embedchain.models.data_type import (DataType, DirectDataType,- IndirectDataType, SpecialDataType)+from embedchain.models.data_type import DataType, DirectDataType, IndirectDataType, SpecialDataTypefrom embedchain.telemetry.posthog import AnonymousTelemetryfrom embedchain.utils import detect_datatype, is_valid_json_stringfrom embedchain.vectordb.base import BaseVectorDB@@ -84,6 +83,7 @@ class EmbedChain(JSONSerializable):# Attributes that aren't subclass related.self.user_asks = []+ self.chunker: ChunkerConfig = None# Send anonymous telemetryself._telemetry_props = {"class": self.__class__.__name__}self.telemetry = AnonymousTelemetry(enabled=self.config.collect_metrics)@@ -157,7 +157,11 @@ class EmbedChain(JSONSerializable)::return: source_hash, a md5-hash of the source, in hexadecimal representation.:rtype: str"""- if config is None:+ if config is not None:+ pass+ elif self.chunker is not None:+ config = AddConfig(chunker=self.chunker)+ else:config = AddConfig()try:diff --git a/embedchain/pipeline.py b/embedchain/pipeline.pyindex 7278f6e9d..64e38e8b3 100644--- a/embedchain/pipeline.py+++ b/embedchain/pipeline.py@@ -9,7 +9,7 @@ import requestsimport yamlfrom embedchain import Client-from embedchain.config import PipelineConfig+from embedchain.config import PipelineConfig, ChunkerConfigfrom embedchain.embedchain import CONFIG_DIR, EmbedChainfrom embedchain.embedder.base import BaseEmbedderfrom embedchain.embedder.openai import OpenAIEmbedder@@ -44,6 +44,7 @@ class Pipeline(EmbedChain):yaml_path: str = None,log_level=logging.INFO,auto_deploy: bool = False,+ chunker: ChunkerConfig = None,):"""Initialize a new `App` instance.@@ -84,6 +85,10 @@ class Pipeline(EmbedChain):# pipeline_id from the backendself.id = None+ self.chunker = None+ if chunker:+ self.chunker = ChunkerConfig(**chunker)+self.config = config or PipelineConfig()self.name = self.config.name@@ -366,6 +371,7 @@ class Pipeline(EmbedChain):db_config_data = config_data.get("vectordb", {})embedding_model_config_data = config_data.get("embedding_model", config_data.get("embedder", {}))llm_config_data = config_data.get("llm", {})+ chunker_config_data = config_data.get("chunker", {})pipeline_config = PipelineConfig(**pipeline_config_data)@@ -394,4 +400,5 @@ class Pipeline(EmbedChain):embedding_model=embedding_model,yaml_path=yaml_path,auto_deploy=auto_deploy,+ chunker=chunker_config_data,)diff --git a/embedchain/utils.py b/embedchain/utils.pyindex 87eed4d31..99a01f8b6 100644--- a/embedchain/utils.py+++ b/embedchain/utils.py@@ -350,6 +350,11 @@ def validate_yaml_config(config_data):Optional("deployment_name"): str,},},+ Optional("chunker"): {+ Optional("chunk_size"): int,+ Optional("chunk_overlap"): int,+ Optional("length_function"): str,+ },})