Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add create_element_from_text to Unstructured IO module (#516)

Wendong-Fan3def4e7f8b65
diff --git a/camel/loaders/unstructured_io.py b/camel/loaders/unstructured_io.py
index 2be9c25c7..ec3568a7c 100644
--- a/camel/loaders/unstructured_io.py
+++ b/camel/loaders/unstructured_io.py
@@ -11,9 +11,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-
+import uuid
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+from unstructured.documents.elements import Element
+
class UnstructuredIO:
r"""A class to handle various functionalities provided by the
@@ -71,6 +73,50 @@ class UnstructuredIO:
raise ValueError(f"Require `unstructured>={min_version}`, "
f"you have {__version__}.")
+ def create_element_from_text(
+ self, text: str, element_id: Optional[Union[str,
+ uuid.UUID]] = None,
+ embeddings: Optional[List[float]] = None,
+ filename: Optional[str] = None,
+ file_directory: Optional[str] = None,
+ last_modified: Optional[str] = None,
+ filetype: Optional[str] = None,
+ parent_id: Optional[Union[str, uuid.UUID]] = None) -> Element:
+ r"""Creates a Text element from a given text input, with optional
+ metadata and embeddings.
+
+ Args:
+ text (str): The text content for the element.
+ element_id (Union[str, uuid.UUID], optional): Unique identifier
+ forthe element. Defaults to an empty string.
+ embeddings (Optional[List[float]], optional): A list of float
+ numbers representing the text embeddings. Defaults to `None`.
+ filename (Optional[str], optional): The name of the file the
+ element is associated with. Defaults to `None`.
+ file_directory (Optional[str], optional): The directory path where
+ the file is located. Defaults to `None`.
+ last_modified (Optional[str], optional): The last modified date of
+ the file. Defaults to `None`.
+ filetype (Optional[str], optional): The type of the file. Defaults
+ to `None`.
+ parent_id (Optional[Union[str, uuid.UUID]], optional): The
+ identifier of the parent element. Defaults to `None`.
+
+ Returns:
+ Element: An instance of Text with the provided content and
+ metadata.
+ """
+ from unstructured.documents.elements import ElementMetadata, Text
+
+ metadata = ElementMetadata(filename=filename,
+ file_directory=file_directory,
+ last_modified=last_modified,
+ filetype=filetype, parent_id=parent_id)
+
+ return Text(text=text,
+ element_id=element_id if element_id else str(uuid.uuid4()),
+ metadata=metadata, embeddings=embeddings)
+
def parse_file_or_url(
self,
input_path: str,
diff --git a/test/loaders/test_unstructured_io.py b/test/loaders/test_unstructured_io.py
index 0854fe55f..e7fc90c7f 100644
--- a/test/loaders/test_unstructured_io.py
+++ b/test/loaders/test_unstructured_io.py
@@ -11,6 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import uuid
from typing import Any, Dict, List, Tuple
import pytest
@@ -34,6 +35,41 @@ def test__ensure_unstructured_version(unstructured_instance: UnstructuredIO):
unstructured_instance._ensure_unstructured_version("1.0.0")
+# Test the create_element_from_text method
+def test_create_element_from_text(unstructured_instance: UnstructuredIO):
+ # Input parameters
+ test_text = "Hello, World!"
+ test_id = uuid.uuid4()
+ test_embeddings = [0.1, 0.2, 0.3]
+ test_filename = "testfile.txt"
+ test_directory = "/test/directory"
+ test_modified = "2024-04-01"
+ test_filetype = "txt"
+ test_parent_id = uuid.uuid4()
+
+ # Expected Metadata construction
+ expected_metadata = {
+ "filename": test_filename,
+ "file_directory": test_directory,
+ "last_modified": test_modified,
+ "filetype": test_filetype,
+ "parent_id": test_parent_id
+ }
+
+ # Create the element
+ element = unstructured_instance.create_element_from_text(
+ text=test_text, element_id=test_id, embeddings=test_embeddings,
+ filename=test_filename, file_directory=test_directory,
+ last_modified=test_modified, filetype=test_filetype,
+ parent_id=test_parent_id)
+
+ # Assertions to verify correct element creation
+ assert element.to_dict()['text'] == test_text
+ assert element.to_dict()['element_id'] == test_id
+ assert element.to_dict()['embeddings'] == test_embeddings
+ assert element.to_dict()['metadata'] == expected_metadata
+
+
# Test the parse_file_or_url method
def test_parse_file_or_url(unstructured_instance: UnstructuredIO):
# You can mock the required dependencies and test different scenarios here