🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add create_element_from_text to Unstructured IO module (#516)
Wendong-Fan3def4e7f8b65
diff --git a/camel/loaders/unstructured_io.py b/camel/loaders/unstructured_io.pyindex 2be9c25c7..ec3568a7c 100644--- a/camel/loaders/unstructured_io.py+++ b/camel/loaders/unstructured_io.py@@ -11,9 +11,11 @@# See the License for the specific language governing permissions and# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-+import uuidfrom typing import Any, Dict, List, Literal, Optional, Tuple, Union+from unstructured.documents.elements import Element+class UnstructuredIO:r"""A class to handle various functionalities provided by the@@ -71,6 +73,50 @@ class UnstructuredIO:raise ValueError(f"Require `unstructured>={min_version}`, "f"you have {__version__}.")+ def create_element_from_text(+ self, text: str, element_id: Optional[Union[str,+ uuid.UUID]] = None,+ embeddings: Optional[List[float]] = None,+ filename: Optional[str] = None,+ file_directory: Optional[str] = None,+ last_modified: Optional[str] = None,+ filetype: Optional[str] = None,+ parent_id: Optional[Union[str, uuid.UUID]] = None) -> Element:+ r"""Creates a Text element from a given text input, with optional+ metadata and embeddings.++ Args:+ text (str): The text content for the element.+ element_id (Union[str, uuid.UUID], optional): Unique identifier+ forthe element. Defaults to an empty string.+ embeddings (Optional[List[float]], optional): A list of float+ numbers representing the text embeddings. Defaults to `None`.+ filename (Optional[str], optional): The name of the file the+ element is associated with. Defaults to `None`.+ file_directory (Optional[str], optional): The directory path where+ the file is located. Defaults to `None`.+ last_modified (Optional[str], optional): The last modified date of+ the file. Defaults to `None`.+ filetype (Optional[str], optional): The type of the file. Defaults+ to `None`.+ parent_id (Optional[Union[str, uuid.UUID]], optional): The+ identifier of the parent element. Defaults to `None`.++ Returns:+ Element: An instance of Text with the provided content and+ metadata.+ """+ from unstructured.documents.elements import ElementMetadata, Text++ metadata = ElementMetadata(filename=filename,+ file_directory=file_directory,+ last_modified=last_modified,+ filetype=filetype, parent_id=parent_id)++ return Text(text=text,+ element_id=element_id if element_id else str(uuid.uuid4()),+ metadata=metadata, embeddings=embeddings)+def parse_file_or_url(self,input_path: str,diff --git a/test/loaders/test_unstructured_io.py b/test/loaders/test_unstructured_io.pyindex 0854fe55f..e7fc90c7f 100644--- a/test/loaders/test_unstructured_io.py+++ b/test/loaders/test_unstructured_io.py@@ -11,6 +11,7 @@# See the License for the specific language governing permissions and# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import uuidfrom typing import Any, Dict, List, Tupleimport pytest@@ -34,6 +35,41 @@ def test__ensure_unstructured_version(unstructured_instance: UnstructuredIO):unstructured_instance._ensure_unstructured_version("1.0.0")+# Test the create_element_from_text method+def test_create_element_from_text(unstructured_instance: UnstructuredIO):+ # Input parameters+ test_text = "Hello, World!"+ test_id = uuid.uuid4()+ test_embeddings = [0.1, 0.2, 0.3]+ test_filename = "testfile.txt"+ test_directory = "/test/directory"+ test_modified = "2024-04-01"+ test_filetype = "txt"+ test_parent_id = uuid.uuid4()++ # Expected Metadata construction+ expected_metadata = {+ "filename": test_filename,+ "file_directory": test_directory,+ "last_modified": test_modified,+ "filetype": test_filetype,+ "parent_id": test_parent_id+ }++ # Create the element+ element = unstructured_instance.create_element_from_text(+ text=test_text, element_id=test_id, embeddings=test_embeddings,+ filename=test_filename, file_directory=test_directory,+ last_modified=test_modified, filetype=test_filetype,+ parent_id=test_parent_id)++ # Assertions to verify correct element creation+ assert element.to_dict()['text'] == test_text+ assert element.to_dict()['element_id'] == test_id+ assert element.to_dict()['embeddings'] == test_embeddings+ assert element.to_dict()['metadata'] == expected_metadata++# Test the parse_file_or_url methoddef test_parse_file_or_url(unstructured_instance: UnstructuredIO):# You can mock the required dependencies and test different scenarios here