The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
[improvement] update web page default chunk size to 2000 (#1005)
Deven Patelae6f8669014e
diff --git a/embedchain/chunkers/web_page.py b/embedchain/chunkers/web_page.pyindex 253b2b413..5ef7f40df 100644--- a/embedchain/chunkers/web_page.py+++ b/embedchain/chunkers/web_page.py@@ -13,7 +13,7 @@ class WebPageChunker(BaseChunker):def __init__(self, config: Optional[ChunkerConfig] = None):if config is None:- config = ChunkerConfig(chunk_size=500, chunk_overlap=0, length_function=len)+ config = ChunkerConfig(chunk_size=2000, chunk_overlap=0, length_function=len)text_splitter = RecursiveCharacterTextSplitter(chunk_size=config.chunk_size,chunk_overlap=config.chunk_overlap,diff --git a/tests/chunkers/test_chunkers.py b/tests/chunkers/test_chunkers.pyindex 1e725a8a3..e0de99587 100644--- a/tests/chunkers/test_chunkers.py+++ b/tests/chunkers/test_chunkers.py@@ -31,7 +31,7 @@ chunker_common_config = {QnaPairChunker: {"chunk_size": 300, "chunk_overlap": 0, "length_function": len},TableChunker: {"chunk_size": 300, "chunk_overlap": 0, "length_function": len},SitemapChunker: {"chunk_size": 500, "chunk_overlap": 0, "length_function": len},- WebPageChunker: {"chunk_size": 500, "chunk_overlap": 0, "length_function": len},+ WebPageChunker: {"chunk_size": 2000, "chunk_overlap": 0, "length_function": len},XmlChunker: {"chunk_size": 500, "chunk_overlap": 50, "length_function": len},YoutubeVideoChunker: {"chunk_size": 2000, "chunk_overlap": 0, "length_function": len},JSONChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},