Skip to content

armgit / mem0

Public Imported from GitHub
0

The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.

[improvement] update web page default chunk size to 2000 (#1005)

Deven Patelae6f8669014e
diff --git a/embedchain/chunkers/web_page.py b/embedchain/chunkers/web_page.py
index 253b2b413..5ef7f40df 100644
--- a/embedchain/chunkers/web_page.py
+++ b/embedchain/chunkers/web_page.py
@@ -13,7 +13,7 @@ class WebPageChunker(BaseChunker):
def __init__(self, config: Optional[ChunkerConfig] = None):
if config is None:
- config = ChunkerConfig(chunk_size=500, chunk_overlap=0, length_function=len)
+ config = ChunkerConfig(chunk_size=2000, chunk_overlap=0, length_function=len)
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=config.chunk_size,
chunk_overlap=config.chunk_overlap,
diff --git a/tests/chunkers/test_chunkers.py b/tests/chunkers/test_chunkers.py
index 1e725a8a3..e0de99587 100644
--- a/tests/chunkers/test_chunkers.py
+++ b/tests/chunkers/test_chunkers.py
@@ -31,7 +31,7 @@ chunker_common_config = {
QnaPairChunker: {"chunk_size": 300, "chunk_overlap": 0, "length_function": len},
TableChunker: {"chunk_size": 300, "chunk_overlap": 0, "length_function": len},
SitemapChunker: {"chunk_size": 500, "chunk_overlap": 0, "length_function": len},
- WebPageChunker: {"chunk_size": 500, "chunk_overlap": 0, "length_function": len},
+ WebPageChunker: {"chunk_size": 2000, "chunk_overlap": 0, "length_function": len},
XmlChunker: {"chunk_size": 500, "chunk_overlap": 50, "length_function": len},
YoutubeVideoChunker: {"chunk_size": 2000, "chunk_overlap": 0, "length_function": len},
JSONChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},