🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Chunker module enabling custom chunking strategy (#1727)
AveryYay32c9aedbdcbb
diff --git a/camel/models/stub_model.py b/camel/models/stub_model.pyindex 066dc9e57..1487d7c8c 100644--- a/camel/models/stub_model.py+++ b/camel/models/stub_model.py@@ -44,6 +44,31 @@ class StubTokenCounter(BaseTokenCounter):"""return 10+ def encode(self, text: str) -> List[int]:+ r"""Encode text into token IDs for STUB models.++ Args:+ text (str): The text to encode.++ Returns:+ List[int]: List of token IDs.+ """+ # For stub models, just return a list of 0s with length proportional+ # to text length+ return [0] * (len(text) // 4 + 1) # Simple approximation++ def decode(self, token_ids: List[int]) -> str:+ r"""Decode token IDs back to text for STUB models.++ Args:+ token_ids (List[int]): List of token IDs to decode.++ Returns:+ str: Decoded text.+ """+ # For stub models, return a placeholder string+ return "[Stub decoded text]"+class StubModel(BaseModelBackend):r"""A dummy model used for unit tests."""diff --git a/camel/retrievers/vector_retriever.py b/camel/retrievers/vector_retriever.pyindex d51aef3fd..dfa0db504 100644--- a/camel/retrievers/vector_retriever.py+++ b/camel/retrievers/vector_retriever.py@@ -27,6 +27,7 @@ from camel.storages import (VectorRecord,)from camel.utils import Constants+from camel.utils.chunker import BaseChunker, UnstructuredIOChunkerif TYPE_CHECKING:from unstructured.documents.elements import Element@@ -78,6 +79,7 @@ class VectorRetriever(BaseRetriever):should_chunk: bool = True,extra_info: Optional[dict] = None,metadata_filename: Optional[str] = None,+ chunker: Optional[BaseChunker] = None,**kwargs: Any,) -> None:r"""Processes content from local file path, remote URL, string@@ -101,6 +103,12 @@ class VectorRetriever(BaseRetriever):used for storing metadata. Defaults to None.**kwargs (Any): Additional keyword arguments for content parsing."""+ if chunker is None:+ chunker = UnstructuredIOChunker(+ chunk_type=chunk_type,+ max_characters=max_characters,+ metadata_filename=metadata_filename,+ )from unstructured.documents.elements import Elementif isinstance(content, Element):@@ -140,13 +148,7 @@ class VectorRetriever(BaseRetriever):else:# Chunk the content if requiredchunks = (- self.uio.chunk_elements(- chunk_type=chunk_type,- elements=elements,- max_characters=max_characters,- )- if should_chunk- else elements+ chunker.chunk(content=elements) if should_chunk else (elements))# Process chunks in batches and store embeddings@@ -157,6 +159,7 @@ class VectorRetriever(BaseRetriever):)records = []+ offset = 0# Prepare the payload for each vector record, includes the# content path, chunk metadata, and chunk textfor vector, chunk in zip(batch_vectors, batch_chunks):@@ -178,6 +181,7 @@ class VectorRetriever(BaseRetriever):chunk_metadata["metadata"].pop("orig_elements", "")chunk_metadata["extra_info"] = extra_info or {}chunk_text = {"text": str(chunk)}+ chunk_metadata["metadata"]["piece_num"] = i + offset + 1combined_dict = {**content_path_info,**chunk_metadata,@@ -187,6 +191,7 @@ class VectorRetriever(BaseRetriever):records.append(VectorRecord(vector=vector, payload=combined_dict))+ offset += 1self.storage.add(records=records)diff --git a/camel/utils/chunker/__init__.py b/camel/utils/chunker/__init__.pynew file mode 100644index 000000000..c4ed03d53--- /dev/null+++ b/camel/utils/chunker/__init__.py@@ -0,0 +1,22 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from .base import BaseChunker+from .code_chunker import CodeChunker+from .uio_chunker import UnstructuredIOChunker++__all__ = [+ "BaseChunker",+ "CodeChunker",+ "UnstructuredIOChunker",+]diff --git a/camel/utils/chunker/base.py b/camel/utils/chunker/base.pynew file mode 100644index 000000000..39ade2f3e--- /dev/null+++ b/camel/utils/chunker/base.py@@ -0,0 +1,24 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from abc import ABC, abstractmethod+from typing import Any+++class BaseChunker(ABC):+ r"""An abstract base class for all CAMEL chunkers."""++ @abstractmethod+ def chunk(self, content: Any) -> Any:+ r"""Chunk the given content"""+ passdiff --git a/camel/utils/chunker/code_chunker.py b/camel/utils/chunker/code_chunker.pynew file mode 100644index 000000000..67cec02d5--- /dev/null+++ b/camel/utils/chunker/code_chunker.py@@ -0,0 +1,193 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import re+from typing import List, Optional++from unstructured.documents.elements import Element, ElementMetadata++from camel.messages import OpenAIUserMessage+from camel.types import ModelType+from camel.utils import BaseTokenCounter, OpenAITokenCounter++from .base import BaseChunker+++class CodeChunker(BaseChunker):+ r"""A class for chunking code or text while respecting structure+ and token limits.++ This class ensures that structured elements such as functions,+ classes, and regions are not arbitrarily split across chunks.+ It also handles oversized lines and Base64-encoded images.++ Attributes:+ chunk_size (int, optional): The maximum token size per chunk.+ (default: :obj:`8192`)+ token_counter (BaseTokenCounter, optional): The tokenizer used for+ token counting, if `None`, OpenAITokenCounter will be used.+ (default: :obj:`None`)+ remove_image: (bool, optional): If the chunker should skip the images.+ """++ def __init__(+ self,+ chunk_size: int = 8192,+ token_counter: Optional[BaseTokenCounter] = None,+ remove_image: Optional[bool] = True,+ ):+ self.chunk_size = chunk_size+ self.token_counter = (+ token_counter+ if token_counter+ else OpenAITokenCounter(model=ModelType.GPT_4O_MINI)+ )+ self.remove_image = remove_image+ self.struct_pattern = re.compile(+ r'^\s*(?:(def|class|function)\s+\w+|'+ r'(public|private|protected)\s+[\w<>]+\s+\w+\s*\(|'+ r'\b(interface|enum|namespace)\s+\w+|'+ r'#\s*(region|endregion)\b)'+ )+ self.image_pattern = re.compile(+ r'!\[.*?\]\((?:data:image/[^;]+;base64,[a-zA-Z0-9+/]+=*|[^)]+)\)'+ )++ def count_tokens(self, text: str):+ r"""Counts the number of tokens in the given text.++ Args:+ text (str): The input text to be tokenized.++ Returns:+ int: The number of tokens in the input text.+ """+ return self.token_counter.count_tokens_from_messages(+ [OpenAIUserMessage(role="user", name="user", content=text)]+ )++ def _split_oversized(self, line: str) -> List[str]:+ r"""Splits an oversized line into multiple chunks based on token limits++ Args:+ line (str): The oversized line to be split.++ Returns:+ List[str]: A list of smaller chunks after splitting the+ oversized line.+ """+ tokens = self.token_counter.encode(line)+ chunks = []+ buffer = []+ current_count = 0++ for token in tokens:+ buffer.append(token)+ current_count += 1++ if current_count >= self.chunk_size:+ chunks.append(self.token_counter.decode(buffer).strip())+ buffer = []+ current_count = 0++ if buffer:+ chunks.append(self.token_counter.decode(buffer))+ return chunks++ def chunk(self, content: List[str]) -> List[Element]:+ r"""Splits the content into smaller chunks while preserving+ structure and adhering to token constraints.++ Args:+ content (List[str]): The content to be chunked.++ Returns:+ List[str]: A list of chunked text segments.+ """+ content_str = "\n".join(map(str, content))+ chunks = []+ current_chunk: list[str] = []+ current_tokens = 0+ struct_buffer: list[str] = []+ struct_tokens = 0++ for line in content_str.splitlines(keepends=True):+ if self.remove_image:+ if self.image_pattern.match(line):+ continue++ line_tokens = self.count_tokens(line)++ if line_tokens > self.chunk_size:+ if current_chunk:+ chunks.append("".join(current_chunk))+ current_chunk = []+ current_tokens = 0+ chunks.extend(self._split_oversized(line))+ continue++ if self.struct_pattern.match(line):+ if struct_buffer:+ if current_tokens + struct_tokens <= self.chunk_size:+ current_chunk.extend(struct_buffer)+ current_tokens += struct_tokens+ else:+ if current_chunk:+ chunks.append("".join(current_chunk))+ current_chunk = struct_buffer.copy()+ current_tokens = struct_tokens+ struct_buffer = []+ struct_tokens = 0++ struct_buffer.append(line)+ struct_tokens += line_tokens+ else:+ if struct_buffer:+ struct_buffer.append(line)+ struct_tokens += line_tokens+ else:+ if current_tokens + line_tokens > self.chunk_size:+ chunks.append("".join(current_chunk))+ current_chunk = [line]+ current_tokens = line_tokens+ else:+ current_chunk.append(line)+ current_tokens += line_tokens++ if struct_buffer:+ if current_tokens + struct_tokens <= self.chunk_size:+ current_chunk.extend(struct_buffer)+ else:+ if current_chunk:+ chunks.append("".join(current_chunk))+ current_chunk = struct_buffer++ if current_chunk:+ chunks.append("".join(current_chunk))++ final_chunks = []+ for chunk in chunks:+ chunk_token = self.count_tokens(chunk)+ if chunk_token > self.chunk_size:+ final_chunks.extend(self._split_oversized(chunk))+ else:+ final_chunks.append(chunk)++ # TODO: need to reconsider how to correctly form metadata (maybe need+ # to decouple the connection with unstructuredIO)+ chunked_elements = []+ for chunk in final_chunks:+ element = Element(metadata=ElementMetadata())+ element.text = chunk+ chunked_elements.append(element)+ return chunked_elementsdiff --git a/camel/utils/chunker/uio_chunker.py b/camel/utils/chunker/uio_chunker.pynew file mode 100644index 000000000..89c4a594e--- /dev/null+++ b/camel/utils/chunker/uio_chunker.py@@ -0,0 +1,66 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from typing import List, Optional++from unstructured.documents.elements import Element++from camel.loaders import UnstructuredIO+from camel.utils.chunker import BaseChunker+++class UnstructuredIOChunker(BaseChunker):+ r"""A class for chunking text while respecting structure and+ character limits.++ This class ensures that structured elements, such as document sections+ and titles, are not arbitrarily split across chunks. It utilizes the+ `UnstructuredIO` class to process and segment elements while maintaining+ readability and coherence. The chunking method can be adjusted based on+ the provided `chunk_type` parameter.++ Args:+ chunk_type (str, optional): The method used for chunking text.+ (default: :obj:`"chunk_by_title"`)+ max_characters (int, optional): The maximum number of characters+ allowed per chunk. (default: :obj:`500`)+ metadata_filename (Optional[str], optional): An optional filename+ for storing metadata related to chunking. (default: :obj:`None`)+ """++ def __init__(+ self,+ chunk_type: str = "chunk_by_title",+ max_characters: int = 500,+ metadata_filename: Optional[str] = None,+ ):+ self.uio = UnstructuredIO()+ self.chunk_type = chunk_type+ self.max_characters = max_characters+ self.metadata_filename = metadata_filename++ def chunk(self, content: List[Element]) -> List[Element]:+ r"""Splits the content into smaller chunks while preserving+ structure and adhering to token constraints.++ Args:+ content (List[Element]): The content to be chunked.++ Returns:+ List[Element]: A list of chunked text segments.+ """+ return self.uio.chunk_elements(+ chunk_type=self.chunk_type,+ elements=content,+ max_characters=self.max_characters,+ )diff --git a/camel/utils/token_counting.py b/camel/utils/token_counting.pyindex f535ee161..4011c63ac 100644--- a/camel/utils/token_counting.py+++ b/camel/utils/token_counting.py@@ -90,6 +90,30 @@ class BaseTokenCounter(ABC):"""pass+ @abstractmethod+ def encode(self, text: str) -> List[int]:+ r"""Encode text into token IDs.++ Args:+ text (str): The text to encode.++ Returns:+ List[int]: List of token IDs.+ """+ pass++ @abstractmethod+ def decode(self, token_ids: List[int]) -> str:+ r"""Decode token IDs back to text.++ Args:+ token_ids (List[int]): List of token IDs to decode.++ Returns:+ str: Decoded text.+ """+ pass+class OpenAITokenCounter(BaseTokenCounter):def __init__(self, model: UnifiedModelType):@@ -227,6 +251,28 @@ class OpenAITokenCounter(BaseTokenCounter):total = EXTRA_TOKENS + SQUARE_TOKENS * h * wreturn total+ def encode(self, text: str) -> List[int]:+ r"""Encode text into token IDs.++ Args:+ text (str): The text to encode.++ Returns:+ List[int]: List of token IDs.+ """+ return self.encoding.encode(text, disallowed_special=())++ def decode(self, token_ids: List[int]) -> str:+ r"""Decode token IDs back to text.++ Args:+ token_ids (List[int]): List of token IDs to decode.++ Returns:+ str: Decoded text.+ """+ return self.encoding.decode(token_ids)+class AnthropicTokenCounter(BaseTokenCounter):@dependencies_required('anthropic')@@ -266,6 +312,33 @@ class AnthropicTokenCounter(BaseTokenCounter):model=self.model,).input_tokens+ def encode(self, text: str) -> List[int]:+ r"""Encode text into token IDs.++ Args:+ text (str): The text to encode.++ Returns:+ List[int]: List of token IDs.+ """+ raise NotImplementedError(+ "The Anthropic API does not provide direct access to token IDs. "+ "Use count_tokens_from_messages() for token counting instead."+ )++ def decode(self, token_ids: List[int]) -> str:+ r"""Decode token IDs back to text.++ Args:+ token_ids (List[int]): List of token IDs to decode.++ Returns:+ str: Decoded text.+ """+ raise NotImplementedError(+ "The Anthropic API does not provide functionality to decode token IDs."+ )+class LiteLLMTokenCounter(BaseTokenCounter):def __init__(self, model_type: UnifiedModelType):@@ -319,6 +392,32 @@ class LiteLLMTokenCounter(BaseTokenCounter):"""return self.completion_cost(completion_response=response)+ def encode(self, text: str) -> List[int]:+ r"""Encode text into token IDs.++ Args:+ text (str): The text to encode.++ Returns:+ List[int]: List of token IDs.+ """+ from litellm import encoding++ return encoding.encode(text, disallowed_special=())++ def decode(self, token_ids: List[int]) -> str:+ r"""Decode token IDs back to text.++ Args:+ token_ids (List[int]): List of token IDs to decode.++ Returns:+ str: Decoded text.+ """+ from litellm import encoding++ return encoding.decode(token_ids)+class MistralTokenCounter(BaseTokenCounter):def __init__(self, model_type: ModelType):@@ -390,3 +489,37 @@ class MistralTokenCounter(BaseTokenCounter):)return mistral_request++ def encode(self, text: str) -> List[int]:+ r"""Encode text into token IDs.++ Args:+ text (str): The text to encode.++ Returns:+ List[int]: List of token IDs.+ """+ # Use the Mistral tokenizer to encode the text+ return self.tokenizer.encode_chat_completion(+ ChatCompletionRequest(+ model=self.model_type,+ messages=[+ {+ "role": "user",+ "content": text,+ }+ ],+ )+ )++ def decode(self, token_ids: List[int]) -> str:+ r"""Decode token IDs back to text.++ Args:+ token_ids (List[int]): List of token IDs to decode.++ Returns:+ str: Decoded text.+ """+ # Use the Mistral tokenizer to decode the tokens+ return self.tokenizer.decode(token_ids)