Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Chunker module enabling custom chunking strategy (#1727)

AveryYay32c9aedbdcbb
diff --git a/camel/models/stub_model.py b/camel/models/stub_model.py
index 066dc9e57..1487d7c8c 100644
--- a/camel/models/stub_model.py
+++ b/camel/models/stub_model.py
@@ -44,6 +44,31 @@ class StubTokenCounter(BaseTokenCounter):
"""
return 10
+ def encode(self, text: str) -> List[int]:
+ r"""Encode text into token IDs for STUB models.
+
+ Args:
+ text (str): The text to encode.
+
+ Returns:
+ List[int]: List of token IDs.
+ """
+ # For stub models, just return a list of 0s with length proportional
+ # to text length
+ return [0] * (len(text) // 4 + 1) # Simple approximation
+
+ def decode(self, token_ids: List[int]) -> str:
+ r"""Decode token IDs back to text for STUB models.
+
+ Args:
+ token_ids (List[int]): List of token IDs to decode.
+
+ Returns:
+ str: Decoded text.
+ """
+ # For stub models, return a placeholder string
+ return "[Stub decoded text]"
+
class StubModel(BaseModelBackend):
r"""A dummy model used for unit tests."""
diff --git a/camel/retrievers/vector_retriever.py b/camel/retrievers/vector_retriever.py
index d51aef3fd..dfa0db504 100644
--- a/camel/retrievers/vector_retriever.py
+++ b/camel/retrievers/vector_retriever.py
@@ -27,6 +27,7 @@ from camel.storages import (
VectorRecord,
)
from camel.utils import Constants
+from camel.utils.chunker import BaseChunker, UnstructuredIOChunker
if TYPE_CHECKING:
from unstructured.documents.elements import Element
@@ -78,6 +79,7 @@ class VectorRetriever(BaseRetriever):
should_chunk: bool = True,
extra_info: Optional[dict] = None,
metadata_filename: Optional[str] = None,
+ chunker: Optional[BaseChunker] = None,
**kwargs: Any,
) -> None:
r"""Processes content from local file path, remote URL, string
@@ -101,6 +103,12 @@ class VectorRetriever(BaseRetriever):
used for storing metadata. Defaults to None.
**kwargs (Any): Additional keyword arguments for content parsing.
"""
+ if chunker is None:
+ chunker = UnstructuredIOChunker(
+ chunk_type=chunk_type,
+ max_characters=max_characters,
+ metadata_filename=metadata_filename,
+ )
from unstructured.documents.elements import Element
if isinstance(content, Element):
@@ -140,13 +148,7 @@ class VectorRetriever(BaseRetriever):
else:
# Chunk the content if required
chunks = (
- self.uio.chunk_elements(
- chunk_type=chunk_type,
- elements=elements,
- max_characters=max_characters,
- )
- if should_chunk
- else elements
+ chunker.chunk(content=elements) if should_chunk else (elements)
)
# Process chunks in batches and store embeddings
@@ -157,6 +159,7 @@ class VectorRetriever(BaseRetriever):
)
records = []
+ offset = 0
# Prepare the payload for each vector record, includes the
# content path, chunk metadata, and chunk text
for vector, chunk in zip(batch_vectors, batch_chunks):
@@ -178,6 +181,7 @@ class VectorRetriever(BaseRetriever):
chunk_metadata["metadata"].pop("orig_elements", "")
chunk_metadata["extra_info"] = extra_info or {}
chunk_text = {"text": str(chunk)}
+ chunk_metadata["metadata"]["piece_num"] = i + offset + 1
combined_dict = {
**content_path_info,
**chunk_metadata,
@@ -187,6 +191,7 @@ class VectorRetriever(BaseRetriever):
records.append(
VectorRecord(vector=vector, payload=combined_dict)
)
+ offset += 1
self.storage.add(records=records)
diff --git a/camel/utils/chunker/__init__.py b/camel/utils/chunker/__init__.py
new file mode 100644
index 000000000..c4ed03d53
--- /dev/null
+++ b/camel/utils/chunker/__init__.py
@@ -0,0 +1,22 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .base import BaseChunker
+from .code_chunker import CodeChunker
+from .uio_chunker import UnstructuredIOChunker
+
+__all__ = [
+ "BaseChunker",
+ "CodeChunker",
+ "UnstructuredIOChunker",
+]
diff --git a/camel/utils/chunker/base.py b/camel/utils/chunker/base.py
new file mode 100644
index 000000000..39ade2f3e
--- /dev/null
+++ b/camel/utils/chunker/base.py
@@ -0,0 +1,24 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BaseChunker(ABC):
+ r"""An abstract base class for all CAMEL chunkers."""
+
+ @abstractmethod
+ def chunk(self, content: Any) -> Any:
+ r"""Chunk the given content"""
+ pass
diff --git a/camel/utils/chunker/code_chunker.py b/camel/utils/chunker/code_chunker.py
new file mode 100644
index 000000000..67cec02d5
--- /dev/null
+++ b/camel/utils/chunker/code_chunker.py
@@ -0,0 +1,193 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import re
+from typing import List, Optional
+
+from unstructured.documents.elements import Element, ElementMetadata
+
+from camel.messages import OpenAIUserMessage
+from camel.types import ModelType
+from camel.utils import BaseTokenCounter, OpenAITokenCounter
+
+from .base import BaseChunker
+
+
+class CodeChunker(BaseChunker):
+ r"""A class for chunking code or text while respecting structure
+ and token limits.
+
+ This class ensures that structured elements such as functions,
+ classes, and regions are not arbitrarily split across chunks.
+ It also handles oversized lines and Base64-encoded images.
+
+ Attributes:
+ chunk_size (int, optional): The maximum token size per chunk.
+ (default: :obj:`8192`)
+ token_counter (BaseTokenCounter, optional): The tokenizer used for
+ token counting, if `None`, OpenAITokenCounter will be used.
+ (default: :obj:`None`)
+ remove_image: (bool, optional): If the chunker should skip the images.
+ """
+
+ def __init__(
+ self,
+ chunk_size: int = 8192,
+ token_counter: Optional[BaseTokenCounter] = None,
+ remove_image: Optional[bool] = True,
+ ):
+ self.chunk_size = chunk_size
+ self.token_counter = (
+ token_counter
+ if token_counter
+ else OpenAITokenCounter(model=ModelType.GPT_4O_MINI)
+ )
+ self.remove_image = remove_image
+ self.struct_pattern = re.compile(
+ r'^\s*(?:(def|class|function)\s+\w+|'
+ r'(public|private|protected)\s+[\w<>]+\s+\w+\s*\(|'
+ r'\b(interface|enum|namespace)\s+\w+|'
+ r'#\s*(region|endregion)\b)'
+ )
+ self.image_pattern = re.compile(
+ r'!\[.*?\]\((?:data:image/[^;]+;base64,[a-zA-Z0-9+/]+=*|[^)]+)\)'
+ )
+
+ def count_tokens(self, text: str):
+ r"""Counts the number of tokens in the given text.
+
+ Args:
+ text (str): The input text to be tokenized.
+
+ Returns:
+ int: The number of tokens in the input text.
+ """
+ return self.token_counter.count_tokens_from_messages(
+ [OpenAIUserMessage(role="user", name="user", content=text)]
+ )
+
+ def _split_oversized(self, line: str) -> List[str]:
+ r"""Splits an oversized line into multiple chunks based on token limits
+
+ Args:
+ line (str): The oversized line to be split.
+
+ Returns:
+ List[str]: A list of smaller chunks after splitting the
+ oversized line.
+ """
+ tokens = self.token_counter.encode(line)
+ chunks = []
+ buffer = []
+ current_count = 0
+
+ for token in tokens:
+ buffer.append(token)
+ current_count += 1
+
+ if current_count >= self.chunk_size:
+ chunks.append(self.token_counter.decode(buffer).strip())
+ buffer = []
+ current_count = 0
+
+ if buffer:
+ chunks.append(self.token_counter.decode(buffer))
+ return chunks
+
+ def chunk(self, content: List[str]) -> List[Element]:
+ r"""Splits the content into smaller chunks while preserving
+ structure and adhering to token constraints.
+
+ Args:
+ content (List[str]): The content to be chunked.
+
+ Returns:
+ List[str]: A list of chunked text segments.
+ """
+ content_str = "\n".join(map(str, content))
+ chunks = []
+ current_chunk: list[str] = []
+ current_tokens = 0
+ struct_buffer: list[str] = []
+ struct_tokens = 0
+
+ for line in content_str.splitlines(keepends=True):
+ if self.remove_image:
+ if self.image_pattern.match(line):
+ continue
+
+ line_tokens = self.count_tokens(line)
+
+ if line_tokens > self.chunk_size:
+ if current_chunk:
+ chunks.append("".join(current_chunk))
+ current_chunk = []
+ current_tokens = 0
+ chunks.extend(self._split_oversized(line))
+ continue
+
+ if self.struct_pattern.match(line):
+ if struct_buffer:
+ if current_tokens + struct_tokens <= self.chunk_size:
+ current_chunk.extend(struct_buffer)
+ current_tokens += struct_tokens
+ else:
+ if current_chunk:
+ chunks.append("".join(current_chunk))
+ current_chunk = struct_buffer.copy()
+ current_tokens = struct_tokens
+ struct_buffer = []
+ struct_tokens = 0
+
+ struct_buffer.append(line)
+ struct_tokens += line_tokens
+ else:
+ if struct_buffer:
+ struct_buffer.append(line)
+ struct_tokens += line_tokens
+ else:
+ if current_tokens + line_tokens > self.chunk_size:
+ chunks.append("".join(current_chunk))
+ current_chunk = [line]
+ current_tokens = line_tokens
+ else:
+ current_chunk.append(line)
+ current_tokens += line_tokens
+
+ if struct_buffer:
+ if current_tokens + struct_tokens <= self.chunk_size:
+ current_chunk.extend(struct_buffer)
+ else:
+ if current_chunk:
+ chunks.append("".join(current_chunk))
+ current_chunk = struct_buffer
+
+ if current_chunk:
+ chunks.append("".join(current_chunk))
+
+ final_chunks = []
+ for chunk in chunks:
+ chunk_token = self.count_tokens(chunk)
+ if chunk_token > self.chunk_size:
+ final_chunks.extend(self._split_oversized(chunk))
+ else:
+ final_chunks.append(chunk)
+
+ # TODO: need to reconsider how to correctly form metadata (maybe need
+ # to decouple the connection with unstructuredIO)
+ chunked_elements = []
+ for chunk in final_chunks:
+ element = Element(metadata=ElementMetadata())
+ element.text = chunk
+ chunked_elements.append(element)
+ return chunked_elements
diff --git a/camel/utils/chunker/uio_chunker.py b/camel/utils/chunker/uio_chunker.py
new file mode 100644
index 000000000..89c4a594e
--- /dev/null
+++ b/camel/utils/chunker/uio_chunker.py
@@ -0,0 +1,66 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import List, Optional
+
+from unstructured.documents.elements import Element
+
+from camel.loaders import UnstructuredIO
+from camel.utils.chunker import BaseChunker
+
+
+class UnstructuredIOChunker(BaseChunker):
+ r"""A class for chunking text while respecting structure and
+ character limits.
+
+ This class ensures that structured elements, such as document sections
+ and titles, are not arbitrarily split across chunks. It utilizes the
+ `UnstructuredIO` class to process and segment elements while maintaining
+ readability and coherence. The chunking method can be adjusted based on
+ the provided `chunk_type` parameter.
+
+ Args:
+ chunk_type (str, optional): The method used for chunking text.
+ (default: :obj:`"chunk_by_title"`)
+ max_characters (int, optional): The maximum number of characters
+ allowed per chunk. (default: :obj:`500`)
+ metadata_filename (Optional[str], optional): An optional filename
+ for storing metadata related to chunking. (default: :obj:`None`)
+ """
+
+ def __init__(
+ self,
+ chunk_type: str = "chunk_by_title",
+ max_characters: int = 500,
+ metadata_filename: Optional[str] = None,
+ ):
+ self.uio = UnstructuredIO()
+ self.chunk_type = chunk_type
+ self.max_characters = max_characters
+ self.metadata_filename = metadata_filename
+
+ def chunk(self, content: List[Element]) -> List[Element]:
+ r"""Splits the content into smaller chunks while preserving
+ structure and adhering to token constraints.
+
+ Args:
+ content (List[Element]): The content to be chunked.
+
+ Returns:
+ List[Element]: A list of chunked text segments.
+ """
+ return self.uio.chunk_elements(
+ chunk_type=self.chunk_type,
+ elements=content,
+ max_characters=self.max_characters,
+ )
diff --git a/camel/utils/token_counting.py b/camel/utils/token_counting.py
index f535ee161..4011c63ac 100644
--- a/camel/utils/token_counting.py
+++ b/camel/utils/token_counting.py
@@ -90,6 +90,30 @@ class BaseTokenCounter(ABC):
"""
pass
+ @abstractmethod
+ def encode(self, text: str) -> List[int]:
+ r"""Encode text into token IDs.
+
+ Args:
+ text (str): The text to encode.
+
+ Returns:
+ List[int]: List of token IDs.
+ """
+ pass
+
+ @abstractmethod
+ def decode(self, token_ids: List[int]) -> str:
+ r"""Decode token IDs back to text.
+
+ Args:
+ token_ids (List[int]): List of token IDs to decode.
+
+ Returns:
+ str: Decoded text.
+ """
+ pass
+
class OpenAITokenCounter(BaseTokenCounter):
def __init__(self, model: UnifiedModelType):
@@ -227,6 +251,28 @@ class OpenAITokenCounter(BaseTokenCounter):
total = EXTRA_TOKENS + SQUARE_TOKENS * h * w
return total
+ def encode(self, text: str) -> List[int]:
+ r"""Encode text into token IDs.
+
+ Args:
+ text (str): The text to encode.
+
+ Returns:
+ List[int]: List of token IDs.
+ """
+ return self.encoding.encode(text, disallowed_special=())
+
+ def decode(self, token_ids: List[int]) -> str:
+ r"""Decode token IDs back to text.
+
+ Args:
+ token_ids (List[int]): List of token IDs to decode.
+
+ Returns:
+ str: Decoded text.
+ """
+ return self.encoding.decode(token_ids)
+
class AnthropicTokenCounter(BaseTokenCounter):
@dependencies_required('anthropic')
@@ -266,6 +312,33 @@ class AnthropicTokenCounter(BaseTokenCounter):
model=self.model,
).input_tokens
+ def encode(self, text: str) -> List[int]:
+ r"""Encode text into token IDs.
+
+ Args:
+ text (str): The text to encode.
+
+ Returns:
+ List[int]: List of token IDs.
+ """
+ raise NotImplementedError(
+ "The Anthropic API does not provide direct access to token IDs. "
+ "Use count_tokens_from_messages() for token counting instead."
+ )
+
+ def decode(self, token_ids: List[int]) -> str:
+ r"""Decode token IDs back to text.
+
+ Args:
+ token_ids (List[int]): List of token IDs to decode.
+
+ Returns:
+ str: Decoded text.
+ """
+ raise NotImplementedError(
+ "The Anthropic API does not provide functionality to decode token IDs."
+ )
+
class LiteLLMTokenCounter(BaseTokenCounter):
def __init__(self, model_type: UnifiedModelType):
@@ -319,6 +392,32 @@ class LiteLLMTokenCounter(BaseTokenCounter):
"""
return self.completion_cost(completion_response=response)
+ def encode(self, text: str) -> List[int]:
+ r"""Encode text into token IDs.
+
+ Args:
+ text (str): The text to encode.
+
+ Returns:
+ List[int]: List of token IDs.
+ """
+ from litellm import encoding
+
+ return encoding.encode(text, disallowed_special=())
+
+ def decode(self, token_ids: List[int]) -> str:
+ r"""Decode token IDs back to text.
+
+ Args:
+ token_ids (List[int]): List of token IDs to decode.
+
+ Returns:
+ str: Decoded text.
+ """
+ from litellm import encoding
+
+ return encoding.decode(token_ids)
+
class MistralTokenCounter(BaseTokenCounter):
def __init__(self, model_type: ModelType):
@@ -390,3 +489,37 @@ class MistralTokenCounter(BaseTokenCounter):
)
return mistral_request
+
+ def encode(self, text: str) -> List[int]:
+ r"""Encode text into token IDs.
+
+ Args:
+ text (str): The text to encode.
+
+ Returns:
+ List[int]: List of token IDs.
+ """
+ # Use the Mistral tokenizer to encode the text
+ return self.tokenizer.encode_chat_completion(
+ ChatCompletionRequest(
+ model=self.model_type,
+ messages=[
+ {
+ "role": "user",
+ "content": text,
+ }
+ ],
+ )
+ )
+
+ def decode(self, token_ids: List[int]) -> str:
+ r"""Decode token IDs back to text.
+
+ Args:
+ token_ids (List[int]): List of token IDs to decode.
+
+ Returns:
+ str: Decoded text.
+ """
+ # Use the Mistral tokenizer to decode the tokens
+ return self.tokenizer.decode(token_ids)