š« CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: knowledge graph storage (#450)
Wendong-Fan70d5e7819c6b
diff --git a/camel/storages/__init__.py b/camel/storages/__init__.pyindex dff79f326..37e82f61a 100644--- a/camel/storages/__init__.py+++ b/camel/storages/__init__.py@@ -12,6 +12,8 @@# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from .graph_storages.base import BaseGraphStorage+from .graph_storages.neo4j_graph import Neo4jGraphfrom .key_value_storages.base import BaseKeyValueStoragefrom .key_value_storages.in_memory import InMemoryKeyValueStoragefrom .key_value_storages.json import JsonStorage@@ -34,4 +36,6 @@ __all__ = ['VectorDBQueryResult','QdrantStorage','MilvusStorage',+ 'BaseGraphStorage',+ 'Neo4jGraph',]diff --git a/camel/storages/graph_storages/__init__.py b/camel/storages/graph_storages/__init__.pynew file mode 100644index 000000000..ae829accc--- /dev/null+++ b/camel/storages/graph_storages/__init__.py@@ -0,0 +1,23 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========++from .base import BaseGraphStorage+from .graph_element import GraphElement+from .neo4j_graph import Neo4jGraph++__all__ = [+ 'BaseGraphStorage',+ 'GraphElement',+ 'Neo4jGraph',+]diff --git a/camel/storages/graph_storages/base.py b/camel/storages/graph_storages/base.pynew file mode 100644index 000000000..1be569c9c--- /dev/null+++ b/camel/storages/graph_storages/base.py@@ -0,0 +1,82 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========++from abc import ABC, abstractmethod+from typing import Any, Dict, List, Optional+++class BaseGraphStorage(ABC):+ r"""An abstract base class for graph storage systems."""++ @property+ @abstractmethod+ def get_client(self) -> Any:+ r"""Get the underlying graph storage client."""+ pass++ @property+ @abstractmethod+ def get_schema(self) -> str:+ r"""Get the schema of the graph storage"""+ pass++ @property+ @abstractmethod+ def get_structured_schema(self) -> Dict[str, Any]:+ r"""Get the structured schema of the graph storage"""+ pass++ @abstractmethod+ def refresh_schema(self) -> None:+ r"""Refreshes the graph schema information."""+ pass++ @abstractmethod+ def add_triplet(self, subj: str, obj: str, rel: str) -> None:+ r"""Adds a relationship (triplet) between two entities in the database.++ Args:+ subj (str): The identifier for the subject entity.+ obj (str): The identifier for the object entity.+ rel (str): The relationship between the subject and object.+ """+ pass++ @abstractmethod+ def delete_triplet(self, subj: str, obj: str, rel: str) -> None:+ r"""Deletes a specific triplet from the graph, comprising a subject,+ object and relationship.++ Args:+ subj (str): The identifier for the subject entity.+ obj (str): The identifier for the object entity.+ rel (str): The relationship between the subject and object.+ """+ pass++ @abstractmethod+ def query(+ self, query: str, params: Optional[Dict[str, Any]] = None+ ) -> List[Dict[str, Any]]:+ r"""Query the graph store with statement and parameters.+ Args:+ query (str): The query to be executed.+ params (Optional[Dict[str, Any]]): A dictionary of parameters to+ be used in the query. Defaults to `None`.++ Returns:+ List[Dict[str, Any]]: A list of dictionaries, each+ dictionary represents a row of results from the query.+ """+ passdiff --git a/camel/storages/graph_storages/graph_element.py b/camel/storages/graph_storages/graph_element.pynew file mode 100644index 000000000..cbab1299b--- /dev/null+++ b/camel/storages/graph_storages/graph_element.py@@ -0,0 +1,74 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from __future__ import annotations++from dataclasses import dataclass, field+from typing import List, Union++from unstructured.documents.elements import Element+++@dataclass+class Node:+ r"""Represents a node in a graph with associated properties.++ Attributes:+ id (Union[str, int]): A unique identifier for the node.+ type (str): The type of the relationship.+ properties (dict): Additional properties and metadata associated with+ the node.+ """++ id: Union[str, int]+ type: str = "Node"+ properties: dict = field(default_factory=dict)+++@dataclass+class Relationship:+ r"""Represents a directed relationship between two nodes in a graph.++ Attributes:+ subj (Node): The subject/source node of the relationship.+ obj (Node): The object/target node of the relationship.+ type (str): The type of the relationship.+ properties (dict): Additional properties associated with the+ relationship.+ """++ subj: Node+ obj: Node+ type: str = "Relationship"+ properties: dict = field(default_factory=dict)+++@dataclass+class GraphElement:+ r"""A graph element with lists of nodes and relationships.++ Attributes:+ nodes (List[Node]): A list of nodes in the graph.+ relationships (List[Relationship]): A list of relationships in the+ graph.+ source (Element): The element from which the graph information is+ derived.+ """++ # Allow arbitrary types for Element+ class Config:+ arbitrary_types_allowed = True++ nodes: List[Node]+ relationships: List[Relationship]+ source: Elementdiff --git a/camel/storages/graph_storages/neo4j_graph.py b/camel/storages/graph_storages/neo4j_graph.pynew file mode 100644index 000000000..10be15cc7--- /dev/null+++ b/camel/storages/graph_storages/neo4j_graph.py@@ -0,0 +1,582 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import logging+from hashlib import md5+from typing import Any, Dict, List, Optional++from camel.storages.graph_storages import BaseGraphStorage, GraphElement++logger = logging.getLogger(__name__)++BASE_ENTITY_LABEL = "__Entity__"+EXCLUDED_LABELS = ["Excluded_Label_A", "Excluded_Label_B"]+EXCLUDED_RELS = ["Excluded_Rel_A"]++NODE_PROPERTY_QUERY = """+CALL apoc.meta.data()+YIELD label, other, elementType, type, property+WHERE NOT type = "RELATIONSHIP" AND elementType = "node"+AND NOT label IN $EXCLUDED_LABELS+WITH label AS nodeLabels, collect({property:property, type:type}) AS properties+RETURN {labels: nodeLabels, properties: properties} AS output+"""++REL_PROPERTY_QUERY = """+CALL apoc.meta.data()+YIELD label, other, elementType, type, property+WHERE NOT type = "RELATIONSHIP" AND elementType = "relationship"+AND NOT label IN $EXCLUDED_LABELS+WITH label AS nodeLabels, collect({property:property, type:type}) AS properties+RETURN {type: nodeLabels, properties: properties} AS output+"""++REL_QUERY = """+CALL apoc.meta.data()+YIELD label, other, elementType, type, property+WHERE type = "RELATIONSHIP" AND elementType = "node"+UNWIND other AS other_node+WITH * WHERE NOT label IN $EXCLUDED_LABELS+ AND NOT other_node IN $EXCLUDED_LABELS+RETURN {start: label, type: property, end: toString(other_node)} AS output+"""++INCLUDE_DOCS_QUERY = (+ "MERGE (d:Element {id:$element['element_id']}) "+ "SET d.text = $element['text'] "+ "SET d += $element['metadata'] "+ "WITH d "+)++LIST_LIMIT = 128+++class Neo4jGraph(BaseGraphStorage):+ r"""Provides a connection to a Neo4j database for various graph operations.++ The detailed information about Neo4j is available at:+ `Neo4j https://neo4j.com/docs/getting-started`++ This module refered to the work of Langchian and Llamaindex.++ Args:+ url (str): The URL of the Neo4j database server.+ username (str): The username for database authentication.+ password (str): The password for database authentication.+ database (str): The name of the database to connect to. Defaults to+ `neo4j`.+ timeout (Optional[float]): The timeout for transactions in seconds.+ Useful for terminating long-running queries. Defaults to `None`.+ truncate (bool): A flag to indicate whether to remove lists with more+ than `LIST_LIMIT` elements from results. Defaults to `False`.+ """++ def __init__(+ self,+ url: str,+ username: str,+ password: str,+ database: str = "neo4j",+ timeout: Optional[float] = None,+ truncate: bool = False,+ ) -> None:+ r"""Create a new Neo4j graph instance."""+ try:+ import neo4j+ except ImportError:+ raise ValueError(+ "Could not import neo4j python package. "+ "Please install it with `pip install neo4j`."+ )++ self.driver = neo4j.GraphDatabase.driver(url, auth=(username, password))+ self.database = database+ self.timeout = timeout+ self.truncate = truncate+ self.schema: str = ""+ self.structured_schema: Dict[str, Any] = {}++ # Verify connection+ try:+ self.driver.verify_connectivity()+ except neo4j.exceptions.ServiceUnavailable:+ raise ValueError(+ "Could not connect to Neo4j database. "+ "Please ensure that the url is correct"+ )+ except neo4j.exceptions.AuthError:+ raise ValueError(+ "Could not connect to Neo4j database. "+ "Please ensure that the username and password are correct"+ )+ # Set schema+ try:+ self.refresh_schema()+ except neo4j.exceptions.ClientError:+ raise ValueError(+ "Could not use APOC procedures. "+ "Please ensure the APOC plugin is installed in Neo4j and that "+ "'apoc.meta.data()' is allowed in Neo4j configuration "+ )++ @property+ def get_client(self) -> Any:+ r"""Get the underlying graph storage client."""+ return self.driver++ @property+ def get_schema(self, refresh: bool = False) -> str:+ r"""Retrieve the schema of the Neo4jGraph store.++ Args:+ refresh (bool): A flag indicating whether to forcibly refresh the+ schema from the Neo4jGraph store regardless of whether it is+ already cached. Defaults to `False`.++ Returns:+ str: The schema of the Neo4jGraph store.+ """+ if self.schema and not refresh:+ return self.schema+ self.refresh_schema()+ logger.debug(f"get_schema() schema:\n{self.schema}")+ return self.schema++ @property+ def get_structured_schema(self) -> Dict[str, Any]:+ r"""Returns the structured schema of the graph++ Returns:+ Dict[str, Any]: The structured schema of the graph.+ """+ return self.structured_schema++ def _value_truncate(self, raw_value: Any) -> Any:+ r"""Truncates the input raw value by removing entries that is+ dictionary or list with values resembling embeddings and containing+ more than `LIST_LIMIT` elements. This method aims to reduce unnecessary+ computational cost and noise in scenarios where such detailed data+ structures are not needed. If the input value is not dictionary or+ list then give the raw value back.++ Args:+ raw_value (Any): The raw value to be truncated.++ Returns:+ Any: The truncated value, with embedding-like+ dictionaries and oversized lists handled.+ """+ if isinstance(raw_value, dict):+ new_dict = {}+ for key, value in raw_value.items():+ if isinstance(value, dict):+ truncated_value = self._value_truncate(value)+ # Check if the truncated value is not None+ if truncated_value is not None:+ new_dict[key] = truncated_value+ elif isinstance(value, list):+ if len(value) < LIST_LIMIT:+ truncated_value = self._value_truncate(value)+ # Check if the truncated value is not None+ if truncated_value is not None:+ new_dict[key] = truncated_value+ # Do not include the key if the list is oversized+ else:+ new_dict[key] = value+ return new_dict+ elif isinstance(raw_value, list):+ if len(raw_value) < LIST_LIMIT:+ return [+ self._value_truncate(item)+ for item in raw_value+ if self._value_truncate(item) is not None+ ]+ else:+ return None+ else:+ return raw_value++ def query(+ self, query: str, params: Optional[Dict[str, Any]] = None+ ) -> List[Dict[str, Any]]:+ r"""Executes a Neo4j Cypher declarative query in a database.++ Args:+ query (str): The Cypher query to be executed.+ params (Optional[Dict[str, Any]]): A dictionary of parameters to+ be used in the query. Defaults to `None`.++ Returns:+ List[Dict[str, Any]]: A list of dictionaries, each+ dictionary represents a row of results from the Cypher query.++ Raises:+ ValueError: If the executed Cypher query syntax is invalid.+ """+ from neo4j import Query+ from neo4j.exceptions import CypherSyntaxError++ if params is None:+ params = {}++ with self.driver.session(database=self.database) as session:+ try:+ data = session.run(+ Query(text=query, timeout=self.timeout), params+ )+ json_data = [r.data() for r in data]+ if self.truncate:+ json_data = [self._value_truncate(el) for el in json_data]+ return json_data+ except CypherSyntaxError as e:+ raise ValueError(+ f"Generated Cypher Statement is not valid\n{e}"+ )++ def refresh_schema(self) -> None:+ r"""Refreshes the Neo4j graph schema information by querying the+ database for node properties, relationship properties, and+ relationships.+ """+ from neo4j.exceptions import ClientError++ # Extract schema elements from the database+ node_properties = [+ el["output"]+ for el in self.query(+ NODE_PROPERTY_QUERY,+ params={+ "EXCLUDED_LABELS": [*EXCLUDED_LABELS, BASE_ENTITY_LABEL]+ },+ )+ ]+ rel_properties = [+ el["output"]+ for el in self.query(+ REL_PROPERTY_QUERY, params={"EXCLUDED_LABELS": EXCLUDED_RELS}+ )+ ]+ relationships = [+ el["output"]+ for el in self.query(+ REL_QUERY,+ params={+ "EXCLUDED_LABELS": [*EXCLUDED_LABELS, BASE_ENTITY_LABEL]+ },+ )+ ]++ # Get constraints & indexes+ try:+ constraint = self.query("SHOW CONSTRAINTS")+ index = self.query("SHOW INDEXES YIELD *")+ except (+ ClientError+ ): # Read-only user might not have access to schema information+ constraint = []+ index = []++ self.structured_schema = {+ "node_props": {+ el["labels"]: el["properties"] for el in node_properties+ },+ "rel_props": {+ el["type"]: el["properties"] for el in rel_properties+ },+ "relationships": relationships,+ "metadata": {"constraint": constraint, "index": index},+ }++ # Format node properties+ formatted_node_props = []+ for el in node_properties:+ props_str = ", ".join(+ [+ f"{prop['property']}: {prop['type']}"+ for prop in el["properties"]+ ]+ )+ formatted_node_props.append(f"{el['labels']} {{{props_str}}}")++ # Format relationship properties+ formatted_rel_props = []+ for el in rel_properties:+ props_str = ", ".join(+ [+ f"{prop['property']}: {prop['type']}"+ for prop in el["properties"]+ ]+ )+ formatted_rel_props.append(f"{el['type']} {{{props_str}}}")++ # Format relationships+ formatted_rels = [+ f"(:{el['start']})-[:{el['type']}]->(:{el['end']})"+ for el in relationships+ ]++ self.schema = "\n".join(+ [+ "Node properties are the following:",+ ", ".join(formatted_node_props),+ "Relationship properties are the following:",+ ", ".join(formatted_rel_props),+ "The relationships are the following:",+ ", ".join(formatted_rels),+ ]+ )++ def add_triplet(self, subj: str, obj: str, rel: str) -> None:+ r"""Adds a relationship (triplet) between two entities in the database.++ Args:+ subj (str): The identifier for the subject entity.+ obj (str): The identifier for the object entity.+ rel (str): The relationship between the subject and object.+ """+ query = """+ MERGE (n1:`%s` {id:$subj})+ MERGE (n2:`%s` {id:$obj})+ MERGE (n1)-[:`%s`]->(n2)+ """++ prepared_statement = query % (+ BASE_ENTITY_LABEL.replace("_", ""),+ BASE_ENTITY_LABEL.replace("_", ""),+ rel.replace(" ", "_").upper(),+ )++ # Execute the query within a database session+ with self.driver.session(database=self.database) as session:+ session.run(prepared_statement, {"subj": subj, "obj": obj})++ def _delete_rel(self, subj: str, obj: str, rel: str) -> None:+ r"""Deletes a specific relationship between two nodes in the Neo4j+ database.++ Args:+ subj (str): The identifier for the subject entity.+ obj (str): The identifier for the object entity.+ rel (str): The relationship between the subject and object to+ delete.+ """+ with self.driver.session(database=self.database) as session:+ session.run(+ (+ "MATCH (n1:{})-[r:{}]->(n2:{}) WHERE n1.id = $subj AND n2.id"+ " = $obj DELETE r"+ ).format(+ BASE_ENTITY_LABEL.replace("_", ""),+ rel,+ BASE_ENTITY_LABEL.replace("_", ""),+ ),+ {"subj": subj, "obj": obj},+ )++ def _delete_entity(self, entity: str) -> None:+ r"""Deletes an entity from the Neo4j database based on its unique+ identifier.++ Args:+ entity (str): The unique identifier of the entity to be deleted.+ """+ with self.driver.session(database=self.database) as session:+ session.run(+ "MATCH (n:%s) WHERE n.id = $entity DELETE n"+ % BASE_ENTITY_LABEL.replace("_", ""),+ {"entity": entity},+ )++ def _check_edges(self, entity: str) -> bool:+ r"""Checks if the given entity has any relationships in the graph+ database.++ Args:+ entity (str): The unique identifier of the entity to check.++ Returns:+ bool: True if the entity has at least one edge (relationship),+ False otherwise.+ """+ with self.driver.session(database=self.database) as session:+ is_exists_result = session.run(+ "MATCH (n1:%s)--() WHERE n1.id = $entity RETURN count(*)"+ % (BASE_ENTITY_LABEL.replace("_", "")),+ {"entity": entity},+ )+ return bool(list(is_exists_result))++ def delete_triplet(self, subj: str, obj: str, rel: str) -> None:+ r"""Deletes a specific triplet from the graph, comprising a subject,+ object and relationship.++ Args:+ subj (str): The identifier for the subject entity.+ obj (str): The identifier for the object entity.+ rel (str): The relationship between the subject and object.+ """+ self._delete_rel(subj, obj, rel)+ if not self._check_edges(subj):+ self._delete_entity(subj)+ if not self._check_edges(obj):+ self._delete_entity(obj)++ def _get_node_import_query(+ self, base_entity_label: bool, include_source: bool+ ) -> str:+ r"""Constructs a Cypher query string for importing nodes into a Neo4j+ database.++ Args:+ base_entity_label (bool): Flag indicating whether to use a base+ entity label in the MERGE operation.+ include_source (bool): Flag indicating whether to include source+ element information in the query.++ Returns:+ str: A Cypher query string tailored based on the provided flags.+ """+ REL = 'MERGE (d)-[:MENTIONS]->(source) ' if include_source else ''+ if base_entity_label:+ return (+ f"{INCLUDE_DOCS_QUERY if include_source else ''}"+ "UNWIND $data AS row "+ f"MERGE (source:`{BASE_ENTITY_LABEL}` {{id: row.id}}) "+ "SET source += row.properties "+ f"{REL}"+ "WITH source, row "+ "CALL apoc.create.addLabels( source, [row.type] ) YIELD node "+ "RETURN distinct 'done' AS result"+ )+ else:+ return (+ f"{INCLUDE_DOCS_QUERY if include_source else ''}"+ "UNWIND $data AS row "+ "CALL apoc.merge.node([row.type], {id: row.id}, "+ "row.properties, {}) YIELD node "+ f"{'MERGE (d)-[:MENTIONS]->(node) ' if include_source else ''}"+ "RETURN distinct 'done' AS result"+ )++ def _get_rel_import_query(self, base_entity_label: bool) -> str:+ r"""Constructs a Cypher query string for importing relationship into a+ Neo4j database.++ Args:+ base_entity_label (bool): Flag indicating whether to use a base+ entity label in the MERGE operation.++ Returns:+ str: A Cypher query string tailored based on the provided flags.+ """+ if base_entity_label:+ return (+ "UNWIND $data AS row "+ f"MERGE (subj:`{BASE_ENTITY_LABEL}` {{id: row.subj}}) "+ f"MERGE (obj:`{BASE_ENTITY_LABEL}` {{id: row.obj}}) "+ "WITH subj, obj, row "+ "CALL apoc.merge.relationship(subj, row.type, "+ "{}, row.properties, obj) YIELD rel "+ "RETURN distinct 'done'"+ )+ else:+ return (+ "UNWIND $data AS row "+ "CALL apoc.merge.node([row.subj_label], {id: row.subj},"+ "{}, {}) YIELD node as subj "+ "CALL apoc.merge.node([row.obj_label], {id: row.obj},"+ "{}, {}) YIELD node as obj "+ "CALL apoc.merge.relationship(subj, row.type, "+ "{}, row.properties, obj) YIELD rel "+ "RETURN distinct 'done'"+ )++ def add_graph_elements(+ self,+ graph_elements: List[GraphElement],+ include_source: bool = False,+ base_entity_label: bool = False,+ ) -> None:+ r"""Adds nodes and relationships from a list of GraphElement objects+ to the graph storage.++ Args:+ graph_elements (List[GraphElement]): A list of GraphElement+ objects that contain the nodes and relationships to be added+ to the graph. Each GraphElement should encapsulate the+ structure of part of the graph, including nodes,+ relationships, and the source element information.+ include_source (bool, optional): If True, stores the source+ element and links it to nodes in the graph using the MENTIONS+ relationship. This is useful for tracing back the origin of+ data. Merges source elements based on the `id` property from+ the source element metadata if available; otherwise it+ calculates the MD5 hash of `page_content` for merging process.+ Defaults to `False`.+ base_entity_label (bool, optional): If True, each newly created+ node gets a secondary `BASE_ENTITY_LABEL` label, which is+ indexed and improves import speed and performance. Defaults to+ `False`.+ """+ if base_entity_label: # check if constraint already exists+ constraint_exists = any(+ el["labelsOrTypes"] == [BASE_ENTITY_LABEL]+ and el["properties"] == ["id"]+ for el in self.structured_schema.get("metadata", {}).get(+ "constraint", []+ )+ )+ if not constraint_exists:+ # Create constraint+ self.query(+ "CREATE CONSTRAINT IF NOT EXISTS FOR"+ f"(b:{BASE_ENTITY_LABEL}) "+ "REQUIRE b.id IS UNIQUE;"+ )+ self.refresh_schema() # refresh constraint information++ node_import_query = self._get_node_import_query(+ base_entity_label, include_source+ )+ rel_import_query = self._get_rel_import_query(base_entity_label)+ for element in graph_elements:+ if not element.source.to_dict()['element_id']:+ element.source.to_dict()['element_id'] = md5(+ str(element).encode("utf-8")+ ).hexdigest()++ # Import nodes+ self.query(+ node_import_query,+ {+ "data": [el.__dict__ for el in element.nodes],+ "element": element.source.to_dict(),+ },+ )+ # Import relationships+ self.query(+ rel_import_query,+ {+ "data": [+ {+ "subj": el.subj.id,+ "subj_label": el.subj.type,+ "obj": el.obj.id,+ "obj_label": el.obj.type,+ "type": el.type.replace(" ", "_").upper(),+ "properties": el.properties,+ }+ for el in element.relationships+ ]+ },+ )diff --git a/poetry.lock b/poetry.lockindex ca185450d..8fb06d0f0 100644--- a/poetry.lock+++ b/poetry.lock@@ -1,4 +1,4 @@-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.[[package]]name = "accelerate"@@ -2233,7 +2233,6 @@ files = [{file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:9e2addd2d1866fe112bc6f80117bcc6bc25191c5ed1bfbcf9f1386a884252ae8"},{file = "lxml-5.2.1-cp37-cp37m-win32.whl", hash = "sha256:f51969bac61441fd31f028d7b3b45962f3ecebf691a510495e5d2cd8c8092dbd"},{file = "lxml-5.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b0b58fbfa1bf7367dde8a557994e3b1637294be6cf2169810375caf8571a085c"},- {file = "lxml-5.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3e183c6e3298a2ed5af9d7a356ea823bccaab4ec2349dc9ed83999fd289d14d5"},{file = "lxml-5.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:804f74efe22b6a227306dd890eecc4f8c59ff25ca35f1f14e7482bbce96ef10b"},{file = "lxml-5.2.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:08802f0c56ed150cc6885ae0788a321b73505d2263ee56dad84d200cab11c07a"},{file = "lxml-5.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f8c09ed18ecb4ebf23e02b8e7a22a05d6411911e6fabef3a36e4f371f4f2585"},@@ -2739,6 +2738,24 @@ files = [{file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},]+[[package]]+name = "neo4j"+version = "5.20.0"+description = "Neo4j Bolt driver for Python"+optional = true+python-versions = ">=3.7"+files = [+ {file = "neo4j-5.20.0.tar.gz", hash = "sha256:c59e54a0c0fa1f109f1d2fa5293c29c2bb30ba388b4f9dd9656919793c10063a"},+]++[package.dependencies]+pytz = "*"++[package.extras]+numpy = ["numpy (>=1.7.0,<2.0.0)"]+pandas = ["numpy (>=1.7.0,<2.0.0)", "pandas (>=1.1.0,<3.0.0)"]+pyarrow = ["pyarrow (>=1.0.0)"]+[[package]]name = "networkx"version = "3.1"@@ -4197,6 +4214,7 @@ files = [{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},+ {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},@@ -6449,8 +6467,9 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linktesting = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"][extras]-all = ["PyMuPDF", "accelerate", "beautifulsoup4", "datasets", "diffusers", "docx2txt", "googlemaps", "opencv-python", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "requests_oauthlib", "sentence-transformers", "sentencepiece", "soundfile", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]+all = ["PyMuPDF", "accelerate", "beautifulsoup4", "datasets", "diffusers", "docx2txt", "googlemaps", "neo4j", "opencv-python", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "requests_oauthlib", "sentence-transformers", "sentencepiece", "soundfile", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]encoders = ["sentence-transformers"]+graph-storages = ["neo4j"]huggingface-agent = ["accelerate", "datasets", "diffusers", "opencv-python", "sentencepiece", "soundfile", "torch", "transformers"]retrievers = ["rank-bm25"]test = ["mock", "pytest"]@@ -6460,4 +6479,4 @@ vector-databases = ["pymilvus", "qdrant-client"][metadata]lock-version = "2.0"python-versions = ">=3.8.1,<3.12"-content-hash = "6201ff3129978ce0a06bc08c14a875f72303df66ecdd32e94edefce88481941c"+content-hash = "7e3b9130dff8cc64c8bd45726af6bb6f80d34bbcefd5f6f57a1440c5489cde58"diff --git a/pyproject.toml b/pyproject.tomlindex 512e922a6..7d3f47a07 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -67,6 +67,9 @@ sentence-transformers = { version = "^2.2.2", optional = true }qdrant-client = { version = "^1.9.0", optional = true }pymilvus = { version = "^2.4.0", optional = true }+# graph-storages+neo4j = { version = "^5.18.0", optional = true }+# retrieversrank-bm25 = { version = "^0.2.2", optional = true }@@ -110,6 +113,10 @@ vector-databases = ["pymilvus",]+graph-storages = [+ "neo4j",+]+retrievers = ["rank-bm25",]@@ -139,6 +146,8 @@ all = ["pymilvus",# encoders"sentence-transformers",+ # graph-storages+ "neo4j",# retrievers"rank-bm25",]diff --git a/test/storages/graph_storages/test_neo4j_graph.py b/test/storages/graph_storages/test_neo4j_graph.pynew file mode 100644index 000000000..c698291e8--- /dev/null+++ b/test/storages/graph_storages/test_neo4j_graph.py@@ -0,0 +1,261 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from unstructured.documents.elements import Element++from camel.storages import Neo4jGraph+from camel.storages.graph_storages.graph_element import (+ GraphElement,+ Node,+ Relationship,+)+from camel.storages.graph_storages.neo4j_graph import (+ BASE_ENTITY_LABEL,+ NODE_PROPERTY_QUERY,+ REL_PROPERTY_QUERY,+ REL_QUERY,+)++test_data = [+ GraphElement(+ nodes=[+ Node(id="id_subj", type="type_subj"),+ Node(id="id_obj", type="type_obj"),+ ],+ relationships=[+ Relationship(+ subj=Node(id="id_subj", type="type_subj"),+ obj=Node(id="id_obj", type="type_obj"),+ type="type_rel",+ )+ ],+ source=Element(element_id="a04b820b51c760a41415c57c1eef8f08"),+ )+]++url = "neo4j+s://5af77aab.databases.neo4j.io"+username = "neo4j"+password = "SEK_Fx5Bx-BkRwMx6__zM_TOPqXLWEP-czuIZ_u7-zE"+++def test_cypher_return_correct_schema() -> None:+ r"""Test that chain returns direct results.+ Tested graph.query and graph.refresh_schema.+ """+ graph = Neo4jGraph(+ url=url,+ username=username,+ password=password,+ )+ # Delete all nodes in the graph+ graph.query("MATCH (n) DETACH DELETE n")+ # Create two nodes and a relationship+ graph.query("""+ CREATE (la:LabelA {property_a: 'a'})+ CREATE (lb:LabelB)+ CREATE (lc:LabelC)+ MERGE (la)-[:REL_TYPE]-> (lb)+ MERGE (la)-[:REL_TYPE {rel_prop: 'abc'}]-> (lc)+ """)+ # Refresh schema information+ graph.refresh_schema()++ node_properties = graph.query(+ NODE_PROPERTY_QUERY, params={"EXCLUDED_LABELS": [BASE_ENTITY_LABEL]}+ )+ relationships_properties = graph.query(+ REL_PROPERTY_QUERY, params={"EXCLUDED_LABELS": [BASE_ENTITY_LABEL]}+ )+ relationships = graph.query(+ REL_QUERY, params={"EXCLUDED_LABELS": [BASE_ENTITY_LABEL]}+ )++ expected_node_properties = [+ {+ "output": {+ "properties": [{"property": "property_a", "type": "STRING"}],+ "labels": "LabelA",+ }+ }+ ]+ expected_relationships_properties = [+ {+ "output": {+ "type": "REL_TYPE",+ "properties": [{"property": "rel_prop", "type": "STRING"}],+ }+ }+ ]+ expected_relationships = [+ {"output": {"start": "LabelA", "type": "REL_TYPE", "end": "LabelB"}},+ {"output": {"start": "LabelA", "type": "REL_TYPE", "end": "LabelC"}},+ ]++ assert node_properties == expected_node_properties+ assert relationships_properties == expected_relationships_properties+ # Order is not guaranteed with Neo4j returns+ assert (+ sorted(relationships, key=lambda x: x["output"]["end"])+ == expected_relationships+ )+++def test_neo4j_timeout() -> None:+ r"""Test that neo4j uses the timeout correctly."""+ graph = Neo4jGraph(+ url=url, username=username, password=password, timeout=0.1+ )+ try:+ graph.query("UNWIND range(0,100000,1) AS i MERGE (:Foo {id:i})")+ except Exception as e:+ assert (+ e.code # type: ignore[attr-defined]+ == "Neo.ClientError.Transaction."+ "TransactionTimedOutClientConfiguration"+ )+++def test_neo4j_truncate_values() -> None:+ r"""Test that neo4j uses the timeout correctly."""+ graph = Neo4jGraph(+ url=url, username=username, password=password, truncate=True+ )+ # Delete all nodes in the graph+ graph.query("MATCH (n) DETACH DELETE n")+ # Create two nodes and a relationship+ graph.query("""+ CREATE (la:LabelA {property_a: 'a'})+ CREATE (lb:LabelB)+ CREATE (lc:LabelC)+ MERGE (la)-[:REL_TYPE]-> (lb)+ MERGE (la)-[:REL_TYPE {rel_prop: 'abc'}]-> (lc)+ """)+ graph.refresh_schema()++ output = graph.query("RETURN range(0,130,1) AS result")+ assert output == [{}]+++def test_neo4j_add_data() -> None:+ r"""Test that neo4j correctly import graph element."""+ graph = Neo4jGraph(+ url=url, username=username, password=password, truncate=True+ )+ # Delete all nodes in the graph+ graph.query("MATCH (n) DETACH DELETE n")+ # Remove all constraints+ graph.query("CALL apoc.schema.assert({}, {})")+ graph.refresh_schema()+ # Create two nodes and a relationship+ graph.add_graph_elements(test_data)+ output = graph.query(+ "MATCH (n) RETURN labels(n) AS label, count(*) AS count ORDER BY label"+ )+ assert output == [+ {"label": ["type_obj"], "count": 1},+ {"label": ["type_subj"], "count": 1},+ ]+ assert graph.structured_schema["metadata"]["constraint"] == []+++def test_neo4j_add_data_source() -> None:+ r"""Test that neo4j correctly import graph element with source."""+ graph = Neo4jGraph(+ url=url, username=username, password=password, truncate=True+ )+ # Delete all nodes in the graph+ graph.query("MATCH (n) DETACH DELETE n")+ # Remove all constraints+ graph.query("CALL apoc.schema.assert({}, {})")+ graph.refresh_schema()+ # Create two nodes and a relationship+ graph.add_graph_elements(test_data, include_source=True)+ output = graph.query(+ "MATCH (n) RETURN labels(n) AS label, count(*) AS count ORDER BY label"+ )+ assert output == [+ {"label": ["Element"], "count": 1},+ {"label": ["type_obj"], "count": 1},+ {"label": ["type_subj"], "count": 1},+ ]+ assert graph.structured_schema["metadata"]["constraint"] == []+++def test_neo4j_add_data_base() -> None:+ r"""Test that neo4j correctly import graph element with base_entity."""+ graph = Neo4jGraph(+ url=url, username=username, password=password, truncate=True+ )+ # Delete all nodes in the graph+ graph.query("MATCH (n) DETACH DELETE n")+ # Remove all constraints+ graph.query("CALL apoc.schema.assert({}, {})")+ graph.refresh_schema()+ # Create two nodes and a relationship+ graph.add_graph_elements(test_data, base_entity_label=True)+ output = graph.query(+ "MATCH (n) RETURN apoc.coll.sort(labels(n)) AS label, "+ "count(*) AS count ORDER BY label"+ )+ assert output == [+ {"label": [BASE_ENTITY_LABEL, "type_obj"], "count": 1},+ {"label": [BASE_ENTITY_LABEL, "type_subj"], "count": 1},+ ]+ assert graph.structured_schema["metadata"]["constraint"] != []+++def test_neo4j_add_data_base_source() -> None:+ r"""Test that neo4j correctly import graph element with base_entity and+ source."""+ graph = Neo4jGraph(+ url=url, username=username, password=password, truncate=True+ )+ # Delete all nodes in the graph+ graph.query("MATCH (n) DETACH DELETE n")+ # Remove all constraints+ graph.query("CALL apoc.schema.assert({}, {})")+ graph.refresh_schema()+ # Create two nodes and a relationship+ graph.add_graph_elements(+ test_data, base_entity_label=True, include_source=True+ )+ output = graph.query(+ "MATCH (n) RETURN apoc.coll.sort(labels(n)) AS label, "+ "count(*) AS count ORDER BY label"+ )+ assert output == [+ {"label": ["Element"], "count": 1},+ {"label": [BASE_ENTITY_LABEL, "type_obj"], "count": 1},+ {"label": [BASE_ENTITY_LABEL, "type_subj"], "count": 1},+ ]+ assert graph.structured_schema["metadata"]["constraint"] != []+++def test_neo4j_filtering_labels() -> None:+ r"""Test that neo4j correctly filters excluded labels."""+ graph = Neo4jGraph(+ url=url, username=username, password=password, truncate=True+ )+ # Delete all nodes in the graph+ graph.query("MATCH (n) DETACH DELETE n")+ # Remove all constraints+ graph.query("CALL apoc.schema.assert({}, {})")+ graph.query(+ "CREATE (:`Excluded_Label_A`)-[:`Excluded_Rel_A`]->"+ "(:`Excluded_Label_B`)"+ )+ graph.refresh_schema()++ # Assert both are empty+ assert graph.structured_schema["node_props"] == {}+ assert graph.structured_schema["relationships"] == []