Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: knowledge graph storage (#450)

Wendong-Fan70d5e7819c6b
diff --git a/camel/storages/__init__.py b/camel/storages/__init__.py
index dff79f326..37e82f61a 100644
--- a/camel/storages/__init__.py
+++ b/camel/storages/__init__.py
@@ -12,6 +12,8 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from .graph_storages.base import BaseGraphStorage
+from .graph_storages.neo4j_graph import Neo4jGraph
from .key_value_storages.base import BaseKeyValueStorage
from .key_value_storages.in_memory import InMemoryKeyValueStorage
from .key_value_storages.json import JsonStorage
@@ -34,4 +36,6 @@ __all__ = [
'VectorDBQueryResult',
'QdrantStorage',
'MilvusStorage',
+ 'BaseGraphStorage',
+ 'Neo4jGraph',
]
diff --git a/camel/storages/graph_storages/__init__.py b/camel/storages/graph_storages/__init__.py
new file mode 100644
index 000000000..ae829accc
--- /dev/null
+++ b/camel/storages/graph_storages/__init__.py
@@ -0,0 +1,23 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from .base import BaseGraphStorage
+from .graph_element import GraphElement
+from .neo4j_graph import Neo4jGraph
+
+__all__ = [
+ 'BaseGraphStorage',
+ 'GraphElement',
+ 'Neo4jGraph',
+]
diff --git a/camel/storages/graph_storages/base.py b/camel/storages/graph_storages/base.py
new file mode 100644
index 000000000..1be569c9c
--- /dev/null
+++ b/camel/storages/graph_storages/base.py
@@ -0,0 +1,82 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+
+class BaseGraphStorage(ABC):
+ r"""An abstract base class for graph storage systems."""
+
+ @property
+ @abstractmethod
+ def get_client(self) -> Any:
+ r"""Get the underlying graph storage client."""
+ pass
+
+ @property
+ @abstractmethod
+ def get_schema(self) -> str:
+ r"""Get the schema of the graph storage"""
+ pass
+
+ @property
+ @abstractmethod
+ def get_structured_schema(self) -> Dict[str, Any]:
+ r"""Get the structured schema of the graph storage"""
+ pass
+
+ @abstractmethod
+ def refresh_schema(self) -> None:
+ r"""Refreshes the graph schema information."""
+ pass
+
+ @abstractmethod
+ def add_triplet(self, subj: str, obj: str, rel: str) -> None:
+ r"""Adds a relationship (triplet) between two entities in the database.
+
+ Args:
+ subj (str): The identifier for the subject entity.
+ obj (str): The identifier for the object entity.
+ rel (str): The relationship between the subject and object.
+ """
+ pass
+
+ @abstractmethod
+ def delete_triplet(self, subj: str, obj: str, rel: str) -> None:
+ r"""Deletes a specific triplet from the graph, comprising a subject,
+ object and relationship.
+
+ Args:
+ subj (str): The identifier for the subject entity.
+ obj (str): The identifier for the object entity.
+ rel (str): The relationship between the subject and object.
+ """
+ pass
+
+ @abstractmethod
+ def query(
+ self, query: str, params: Optional[Dict[str, Any]] = None
+ ) -> List[Dict[str, Any]]:
+ r"""Query the graph store with statement and parameters.
+ Args:
+ query (str): The query to be executed.
+ params (Optional[Dict[str, Any]]): A dictionary of parameters to
+ be used in the query. Defaults to `None`.
+
+ Returns:
+ List[Dict[str, Any]]: A list of dictionaries, each
+ dictionary represents a row of results from the query.
+ """
+ pass
diff --git a/camel/storages/graph_storages/graph_element.py b/camel/storages/graph_storages/graph_element.py
new file mode 100644
index 000000000..cbab1299b
--- /dev/null
+++ b/camel/storages/graph_storages/graph_element.py
@@ -0,0 +1,74 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Union
+
+from unstructured.documents.elements import Element
+
+
+@dataclass
+class Node:
+ r"""Represents a node in a graph with associated properties.
+
+ Attributes:
+ id (Union[str, int]): A unique identifier for the node.
+ type (str): The type of the relationship.
+ properties (dict): Additional properties and metadata associated with
+ the node.
+ """
+
+ id: Union[str, int]
+ type: str = "Node"
+ properties: dict = field(default_factory=dict)
+
+
+@dataclass
+class Relationship:
+ r"""Represents a directed relationship between two nodes in a graph.
+
+ Attributes:
+ subj (Node): The subject/source node of the relationship.
+ obj (Node): The object/target node of the relationship.
+ type (str): The type of the relationship.
+ properties (dict): Additional properties associated with the
+ relationship.
+ """
+
+ subj: Node
+ obj: Node
+ type: str = "Relationship"
+ properties: dict = field(default_factory=dict)
+
+
+@dataclass
+class GraphElement:
+ r"""A graph element with lists of nodes and relationships.
+
+ Attributes:
+ nodes (List[Node]): A list of nodes in the graph.
+ relationships (List[Relationship]): A list of relationships in the
+ graph.
+ source (Element): The element from which the graph information is
+ derived.
+ """
+
+ # Allow arbitrary types for Element
+ class Config:
+ arbitrary_types_allowed = True
+
+ nodes: List[Node]
+ relationships: List[Relationship]
+ source: Element
diff --git a/camel/storages/graph_storages/neo4j_graph.py b/camel/storages/graph_storages/neo4j_graph.py
new file mode 100644
index 000000000..10be15cc7
--- /dev/null
+++ b/camel/storages/graph_storages/neo4j_graph.py
@@ -0,0 +1,582 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import logging
+from hashlib import md5
+from typing import Any, Dict, List, Optional
+
+from camel.storages.graph_storages import BaseGraphStorage, GraphElement
+
+logger = logging.getLogger(__name__)
+
+BASE_ENTITY_LABEL = "__Entity__"
+EXCLUDED_LABELS = ["Excluded_Label_A", "Excluded_Label_B"]
+EXCLUDED_RELS = ["Excluded_Rel_A"]
+
+NODE_PROPERTY_QUERY = """
+CALL apoc.meta.data()
+YIELD label, other, elementType, type, property
+WHERE NOT type = "RELATIONSHIP" AND elementType = "node"
+AND NOT label IN $EXCLUDED_LABELS
+WITH label AS nodeLabels, collect({property:property, type:type}) AS properties
+RETURN {labels: nodeLabels, properties: properties} AS output
+"""
+
+REL_PROPERTY_QUERY = """
+CALL apoc.meta.data()
+YIELD label, other, elementType, type, property
+WHERE NOT type = "RELATIONSHIP" AND elementType = "relationship"
+AND NOT label IN $EXCLUDED_LABELS
+WITH label AS nodeLabels, collect({property:property, type:type}) AS properties
+RETURN {type: nodeLabels, properties: properties} AS output
+"""
+
+REL_QUERY = """
+CALL apoc.meta.data()
+YIELD label, other, elementType, type, property
+WHERE type = "RELATIONSHIP" AND elementType = "node"
+UNWIND other AS other_node
+WITH * WHERE NOT label IN $EXCLUDED_LABELS
+ AND NOT other_node IN $EXCLUDED_LABELS
+RETURN {start: label, type: property, end: toString(other_node)} AS output
+"""
+
+INCLUDE_DOCS_QUERY = (
+ "MERGE (d:Element {id:$element['element_id']}) "
+ "SET d.text = $element['text'] "
+ "SET d += $element['metadata'] "
+ "WITH d "
+)
+
+LIST_LIMIT = 128
+
+
+class Neo4jGraph(BaseGraphStorage):
+ r"""Provides a connection to a Neo4j database for various graph operations.
+
+ The detailed information about Neo4j is available at:
+ `Neo4j https://neo4j.com/docs/getting-started`
+
+ This module refered to the work of Langchian and Llamaindex.
+
+ Args:
+ url (str): The URL of the Neo4j database server.
+ username (str): The username for database authentication.
+ password (str): The password for database authentication.
+ database (str): The name of the database to connect to. Defaults to
+ `neo4j`.
+ timeout (Optional[float]): The timeout for transactions in seconds.
+ Useful for terminating long-running queries. Defaults to `None`.
+ truncate (bool): A flag to indicate whether to remove lists with more
+ than `LIST_LIMIT` elements from results. Defaults to `False`.
+ """
+
+ def __init__(
+ self,
+ url: str,
+ username: str,
+ password: str,
+ database: str = "neo4j",
+ timeout: Optional[float] = None,
+ truncate: bool = False,
+ ) -> None:
+ r"""Create a new Neo4j graph instance."""
+ try:
+ import neo4j
+ except ImportError:
+ raise ValueError(
+ "Could not import neo4j python package. "
+ "Please install it with `pip install neo4j`."
+ )
+
+ self.driver = neo4j.GraphDatabase.driver(url, auth=(username, password))
+ self.database = database
+ self.timeout = timeout
+ self.truncate = truncate
+ self.schema: str = ""
+ self.structured_schema: Dict[str, Any] = {}
+
+ # Verify connection
+ try:
+ self.driver.verify_connectivity()
+ except neo4j.exceptions.ServiceUnavailable:
+ raise ValueError(
+ "Could not connect to Neo4j database. "
+ "Please ensure that the url is correct"
+ )
+ except neo4j.exceptions.AuthError:
+ raise ValueError(
+ "Could not connect to Neo4j database. "
+ "Please ensure that the username and password are correct"
+ )
+ # Set schema
+ try:
+ self.refresh_schema()
+ except neo4j.exceptions.ClientError:
+ raise ValueError(
+ "Could not use APOC procedures. "
+ "Please ensure the APOC plugin is installed in Neo4j and that "
+ "'apoc.meta.data()' is allowed in Neo4j configuration "
+ )
+
+ @property
+ def get_client(self) -> Any:
+ r"""Get the underlying graph storage client."""
+ return self.driver
+
+ @property
+ def get_schema(self, refresh: bool = False) -> str:
+ r"""Retrieve the schema of the Neo4jGraph store.
+
+ Args:
+ refresh (bool): A flag indicating whether to forcibly refresh the
+ schema from the Neo4jGraph store regardless of whether it is
+ already cached. Defaults to `False`.
+
+ Returns:
+ str: The schema of the Neo4jGraph store.
+ """
+ if self.schema and not refresh:
+ return self.schema
+ self.refresh_schema()
+ logger.debug(f"get_schema() schema:\n{self.schema}")
+ return self.schema
+
+ @property
+ def get_structured_schema(self) -> Dict[str, Any]:
+ r"""Returns the structured schema of the graph
+
+ Returns:
+ Dict[str, Any]: The structured schema of the graph.
+ """
+ return self.structured_schema
+
+ def _value_truncate(self, raw_value: Any) -> Any:
+ r"""Truncates the input raw value by removing entries that is
+ dictionary or list with values resembling embeddings and containing
+ more than `LIST_LIMIT` elements. This method aims to reduce unnecessary
+ computational cost and noise in scenarios where such detailed data
+ structures are not needed. If the input value is not dictionary or
+ list then give the raw value back.
+
+ Args:
+ raw_value (Any): The raw value to be truncated.
+
+ Returns:
+ Any: The truncated value, with embedding-like
+ dictionaries and oversized lists handled.
+ """
+ if isinstance(raw_value, dict):
+ new_dict = {}
+ for key, value in raw_value.items():
+ if isinstance(value, dict):
+ truncated_value = self._value_truncate(value)
+ # Check if the truncated value is not None
+ if truncated_value is not None:
+ new_dict[key] = truncated_value
+ elif isinstance(value, list):
+ if len(value) < LIST_LIMIT:
+ truncated_value = self._value_truncate(value)
+ # Check if the truncated value is not None
+ if truncated_value is not None:
+ new_dict[key] = truncated_value
+ # Do not include the key if the list is oversized
+ else:
+ new_dict[key] = value
+ return new_dict
+ elif isinstance(raw_value, list):
+ if len(raw_value) < LIST_LIMIT:
+ return [
+ self._value_truncate(item)
+ for item in raw_value
+ if self._value_truncate(item) is not None
+ ]
+ else:
+ return None
+ else:
+ return raw_value
+
+ def query(
+ self, query: str, params: Optional[Dict[str, Any]] = None
+ ) -> List[Dict[str, Any]]:
+ r"""Executes a Neo4j Cypher declarative query in a database.
+
+ Args:
+ query (str): The Cypher query to be executed.
+ params (Optional[Dict[str, Any]]): A dictionary of parameters to
+ be used in the query. Defaults to `None`.
+
+ Returns:
+ List[Dict[str, Any]]: A list of dictionaries, each
+ dictionary represents a row of results from the Cypher query.
+
+ Raises:
+ ValueError: If the executed Cypher query syntax is invalid.
+ """
+ from neo4j import Query
+ from neo4j.exceptions import CypherSyntaxError
+
+ if params is None:
+ params = {}
+
+ with self.driver.session(database=self.database) as session:
+ try:
+ data = session.run(
+ Query(text=query, timeout=self.timeout), params
+ )
+ json_data = [r.data() for r in data]
+ if self.truncate:
+ json_data = [self._value_truncate(el) for el in json_data]
+ return json_data
+ except CypherSyntaxError as e:
+ raise ValueError(
+ f"Generated Cypher Statement is not valid\n{e}"
+ )
+
+ def refresh_schema(self) -> None:
+ r"""Refreshes the Neo4j graph schema information by querying the
+ database for node properties, relationship properties, and
+ relationships.
+ """
+ from neo4j.exceptions import ClientError
+
+ # Extract schema elements from the database
+ node_properties = [
+ el["output"]
+ for el in self.query(
+ NODE_PROPERTY_QUERY,
+ params={
+ "EXCLUDED_LABELS": [*EXCLUDED_LABELS, BASE_ENTITY_LABEL]
+ },
+ )
+ ]
+ rel_properties = [
+ el["output"]
+ for el in self.query(
+ REL_PROPERTY_QUERY, params={"EXCLUDED_LABELS": EXCLUDED_RELS}
+ )
+ ]
+ relationships = [
+ el["output"]
+ for el in self.query(
+ REL_QUERY,
+ params={
+ "EXCLUDED_LABELS": [*EXCLUDED_LABELS, BASE_ENTITY_LABEL]
+ },
+ )
+ ]
+
+ # Get constraints & indexes
+ try:
+ constraint = self.query("SHOW CONSTRAINTS")
+ index = self.query("SHOW INDEXES YIELD *")
+ except (
+ ClientError
+ ): # Read-only user might not have access to schema information
+ constraint = []
+ index = []
+
+ self.structured_schema = {
+ "node_props": {
+ el["labels"]: el["properties"] for el in node_properties
+ },
+ "rel_props": {
+ el["type"]: el["properties"] for el in rel_properties
+ },
+ "relationships": relationships,
+ "metadata": {"constraint": constraint, "index": index},
+ }
+
+ # Format node properties
+ formatted_node_props = []
+ for el in node_properties:
+ props_str = ", ".join(
+ [
+ f"{prop['property']}: {prop['type']}"
+ for prop in el["properties"]
+ ]
+ )
+ formatted_node_props.append(f"{el['labels']} {{{props_str}}}")
+
+ # Format relationship properties
+ formatted_rel_props = []
+ for el in rel_properties:
+ props_str = ", ".join(
+ [
+ f"{prop['property']}: {prop['type']}"
+ for prop in el["properties"]
+ ]
+ )
+ formatted_rel_props.append(f"{el['type']} {{{props_str}}}")
+
+ # Format relationships
+ formatted_rels = [
+ f"(:{el['start']})-[:{el['type']}]->(:{el['end']})"
+ for el in relationships
+ ]
+
+ self.schema = "\n".join(
+ [
+ "Node properties are the following:",
+ ", ".join(formatted_node_props),
+ "Relationship properties are the following:",
+ ", ".join(formatted_rel_props),
+ "The relationships are the following:",
+ ", ".join(formatted_rels),
+ ]
+ )
+
+ def add_triplet(self, subj: str, obj: str, rel: str) -> None:
+ r"""Adds a relationship (triplet) between two entities in the database.
+
+ Args:
+ subj (str): The identifier for the subject entity.
+ obj (str): The identifier for the object entity.
+ rel (str): The relationship between the subject and object.
+ """
+ query = """
+ MERGE (n1:`%s` {id:$subj})
+ MERGE (n2:`%s` {id:$obj})
+ MERGE (n1)-[:`%s`]->(n2)
+ """
+
+ prepared_statement = query % (
+ BASE_ENTITY_LABEL.replace("_", ""),
+ BASE_ENTITY_LABEL.replace("_", ""),
+ rel.replace(" ", "_").upper(),
+ )
+
+ # Execute the query within a database session
+ with self.driver.session(database=self.database) as session:
+ session.run(prepared_statement, {"subj": subj, "obj": obj})
+
+ def _delete_rel(self, subj: str, obj: str, rel: str) -> None:
+ r"""Deletes a specific relationship between two nodes in the Neo4j
+ database.
+
+ Args:
+ subj (str): The identifier for the subject entity.
+ obj (str): The identifier for the object entity.
+ rel (str): The relationship between the subject and object to
+ delete.
+ """
+ with self.driver.session(database=self.database) as session:
+ session.run(
+ (
+ "MATCH (n1:{})-[r:{}]->(n2:{}) WHERE n1.id = $subj AND n2.id"
+ " = $obj DELETE r"
+ ).format(
+ BASE_ENTITY_LABEL.replace("_", ""),
+ rel,
+ BASE_ENTITY_LABEL.replace("_", ""),
+ ),
+ {"subj": subj, "obj": obj},
+ )
+
+ def _delete_entity(self, entity: str) -> None:
+ r"""Deletes an entity from the Neo4j database based on its unique
+ identifier.
+
+ Args:
+ entity (str): The unique identifier of the entity to be deleted.
+ """
+ with self.driver.session(database=self.database) as session:
+ session.run(
+ "MATCH (n:%s) WHERE n.id = $entity DELETE n"
+ % BASE_ENTITY_LABEL.replace("_", ""),
+ {"entity": entity},
+ )
+
+ def _check_edges(self, entity: str) -> bool:
+ r"""Checks if the given entity has any relationships in the graph
+ database.
+
+ Args:
+ entity (str): The unique identifier of the entity to check.
+
+ Returns:
+ bool: True if the entity has at least one edge (relationship),
+ False otherwise.
+ """
+ with self.driver.session(database=self.database) as session:
+ is_exists_result = session.run(
+ "MATCH (n1:%s)--() WHERE n1.id = $entity RETURN count(*)"
+ % (BASE_ENTITY_LABEL.replace("_", "")),
+ {"entity": entity},
+ )
+ return bool(list(is_exists_result))
+
+ def delete_triplet(self, subj: str, obj: str, rel: str) -> None:
+ r"""Deletes a specific triplet from the graph, comprising a subject,
+ object and relationship.
+
+ Args:
+ subj (str): The identifier for the subject entity.
+ obj (str): The identifier for the object entity.
+ rel (str): The relationship between the subject and object.
+ """
+ self._delete_rel(subj, obj, rel)
+ if not self._check_edges(subj):
+ self._delete_entity(subj)
+ if not self._check_edges(obj):
+ self._delete_entity(obj)
+
+ def _get_node_import_query(
+ self, base_entity_label: bool, include_source: bool
+ ) -> str:
+ r"""Constructs a Cypher query string for importing nodes into a Neo4j
+ database.
+
+ Args:
+ base_entity_label (bool): Flag indicating whether to use a base
+ entity label in the MERGE operation.
+ include_source (bool): Flag indicating whether to include source
+ element information in the query.
+
+ Returns:
+ str: A Cypher query string tailored based on the provided flags.
+ """
+ REL = 'MERGE (d)-[:MENTIONS]->(source) ' if include_source else ''
+ if base_entity_label:
+ return (
+ f"{INCLUDE_DOCS_QUERY if include_source else ''}"
+ "UNWIND $data AS row "
+ f"MERGE (source:`{BASE_ENTITY_LABEL}` {{id: row.id}}) "
+ "SET source += row.properties "
+ f"{REL}"
+ "WITH source, row "
+ "CALL apoc.create.addLabels( source, [row.type] ) YIELD node "
+ "RETURN distinct 'done' AS result"
+ )
+ else:
+ return (
+ f"{INCLUDE_DOCS_QUERY if include_source else ''}"
+ "UNWIND $data AS row "
+ "CALL apoc.merge.node([row.type], {id: row.id}, "
+ "row.properties, {}) YIELD node "
+ f"{'MERGE (d)-[:MENTIONS]->(node) ' if include_source else ''}"
+ "RETURN distinct 'done' AS result"
+ )
+
+ def _get_rel_import_query(self, base_entity_label: bool) -> str:
+ r"""Constructs a Cypher query string for importing relationship into a
+ Neo4j database.
+
+ Args:
+ base_entity_label (bool): Flag indicating whether to use a base
+ entity label in the MERGE operation.
+
+ Returns:
+ str: A Cypher query string tailored based on the provided flags.
+ """
+ if base_entity_label:
+ return (
+ "UNWIND $data AS row "
+ f"MERGE (subj:`{BASE_ENTITY_LABEL}` {{id: row.subj}}) "
+ f"MERGE (obj:`{BASE_ENTITY_LABEL}` {{id: row.obj}}) "
+ "WITH subj, obj, row "
+ "CALL apoc.merge.relationship(subj, row.type, "
+ "{}, row.properties, obj) YIELD rel "
+ "RETURN distinct 'done'"
+ )
+ else:
+ return (
+ "UNWIND $data AS row "
+ "CALL apoc.merge.node([row.subj_label], {id: row.subj},"
+ "{}, {}) YIELD node as subj "
+ "CALL apoc.merge.node([row.obj_label], {id: row.obj},"
+ "{}, {}) YIELD node as obj "
+ "CALL apoc.merge.relationship(subj, row.type, "
+ "{}, row.properties, obj) YIELD rel "
+ "RETURN distinct 'done'"
+ )
+
+ def add_graph_elements(
+ self,
+ graph_elements: List[GraphElement],
+ include_source: bool = False,
+ base_entity_label: bool = False,
+ ) -> None:
+ r"""Adds nodes and relationships from a list of GraphElement objects
+ to the graph storage.
+
+ Args:
+ graph_elements (List[GraphElement]): A list of GraphElement
+ objects that contain the nodes and relationships to be added
+ to the graph. Each GraphElement should encapsulate the
+ structure of part of the graph, including nodes,
+ relationships, and the source element information.
+ include_source (bool, optional): If True, stores the source
+ element and links it to nodes in the graph using the MENTIONS
+ relationship. This is useful for tracing back the origin of
+ data. Merges source elements based on the `id` property from
+ the source element metadata if available; otherwise it
+ calculates the MD5 hash of `page_content` for merging process.
+ Defaults to `False`.
+ base_entity_label (bool, optional): If True, each newly created
+ node gets a secondary `BASE_ENTITY_LABEL` label, which is
+ indexed and improves import speed and performance. Defaults to
+ `False`.
+ """
+ if base_entity_label: # check if constraint already exists
+ constraint_exists = any(
+ el["labelsOrTypes"] == [BASE_ENTITY_LABEL]
+ and el["properties"] == ["id"]
+ for el in self.structured_schema.get("metadata", {}).get(
+ "constraint", []
+ )
+ )
+ if not constraint_exists:
+ # Create constraint
+ self.query(
+ "CREATE CONSTRAINT IF NOT EXISTS FOR"
+ f"(b:{BASE_ENTITY_LABEL}) "
+ "REQUIRE b.id IS UNIQUE;"
+ )
+ self.refresh_schema() # refresh constraint information
+
+ node_import_query = self._get_node_import_query(
+ base_entity_label, include_source
+ )
+ rel_import_query = self._get_rel_import_query(base_entity_label)
+ for element in graph_elements:
+ if not element.source.to_dict()['element_id']:
+ element.source.to_dict()['element_id'] = md5(
+ str(element).encode("utf-8")
+ ).hexdigest()
+
+ # Import nodes
+ self.query(
+ node_import_query,
+ {
+ "data": [el.__dict__ for el in element.nodes],
+ "element": element.source.to_dict(),
+ },
+ )
+ # Import relationships
+ self.query(
+ rel_import_query,
+ {
+ "data": [
+ {
+ "subj": el.subj.id,
+ "subj_label": el.subj.type,
+ "obj": el.obj.id,
+ "obj_label": el.obj.type,
+ "type": el.type.replace(" ", "_").upper(),
+ "properties": el.properties,
+ }
+ for el in element.relationships
+ ]
+ },
+ )
diff --git a/poetry.lock b/poetry.lock
index ca185450d..8fb06d0f0 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
[[package]]
name = "accelerate"
@@ -2233,7 +2233,6 @@ files = [
{file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:9e2addd2d1866fe112bc6f80117bcc6bc25191c5ed1bfbcf9f1386a884252ae8"},
{file = "lxml-5.2.1-cp37-cp37m-win32.whl", hash = "sha256:f51969bac61441fd31f028d7b3b45962f3ecebf691a510495e5d2cd8c8092dbd"},
{file = "lxml-5.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b0b58fbfa1bf7367dde8a557994e3b1637294be6cf2169810375caf8571a085c"},
- {file = "lxml-5.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3e183c6e3298a2ed5af9d7a356ea823bccaab4ec2349dc9ed83999fd289d14d5"},
{file = "lxml-5.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:804f74efe22b6a227306dd890eecc4f8c59ff25ca35f1f14e7482bbce96ef10b"},
{file = "lxml-5.2.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:08802f0c56ed150cc6885ae0788a321b73505d2263ee56dad84d200cab11c07a"},
{file = "lxml-5.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f8c09ed18ecb4ebf23e02b8e7a22a05d6411911e6fabef3a36e4f371f4f2585"},
@@ -2739,6 +2738,24 @@ files = [
{file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
]
+[[package]]
+name = "neo4j"
+version = "5.20.0"
+description = "Neo4j Bolt driver for Python"
+optional = true
+python-versions = ">=3.7"
+files = [
+ {file = "neo4j-5.20.0.tar.gz", hash = "sha256:c59e54a0c0fa1f109f1d2fa5293c29c2bb30ba388b4f9dd9656919793c10063a"},
+]
+
+[package.dependencies]
+pytz = "*"
+
+[package.extras]
+numpy = ["numpy (>=1.7.0,<2.0.0)"]
+pandas = ["numpy (>=1.7.0,<2.0.0)", "pandas (>=1.1.0,<3.0.0)"]
+pyarrow = ["pyarrow (>=1.0.0)"]
+
[[package]]
name = "networkx"
version = "3.1"
@@ -4197,6 +4214,7 @@ files = [
{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+ {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -6449,8 +6467,9 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.link
testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"]
[extras]
-all = ["PyMuPDF", "accelerate", "beautifulsoup4", "datasets", "diffusers", "docx2txt", "googlemaps", "opencv-python", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "requests_oauthlib", "sentence-transformers", "sentencepiece", "soundfile", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]
+all = ["PyMuPDF", "accelerate", "beautifulsoup4", "datasets", "diffusers", "docx2txt", "googlemaps", "neo4j", "opencv-python", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "requests_oauthlib", "sentence-transformers", "sentencepiece", "soundfile", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]
encoders = ["sentence-transformers"]
+graph-storages = ["neo4j"]
huggingface-agent = ["accelerate", "datasets", "diffusers", "opencv-python", "sentencepiece", "soundfile", "torch", "transformers"]
retrievers = ["rank-bm25"]
test = ["mock", "pytest"]
@@ -6460,4 +6479,4 @@ vector-databases = ["pymilvus", "qdrant-client"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.8.1,<3.12"
-content-hash = "6201ff3129978ce0a06bc08c14a875f72303df66ecdd32e94edefce88481941c"
+content-hash = "7e3b9130dff8cc64c8bd45726af6bb6f80d34bbcefd5f6f57a1440c5489cde58"
diff --git a/pyproject.toml b/pyproject.toml
index 512e922a6..7d3f47a07 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,6 +67,9 @@ sentence-transformers = { version = "^2.2.2", optional = true }
qdrant-client = { version = "^1.9.0", optional = true }
pymilvus = { version = "^2.4.0", optional = true }
+# graph-storages
+neo4j = { version = "^5.18.0", optional = true }
+
# retrievers
rank-bm25 = { version = "^0.2.2", optional = true }
@@ -110,6 +113,10 @@ vector-databases = [
"pymilvus",
]
+graph-storages = [
+ "neo4j",
+]
+
retrievers = [
"rank-bm25",
]
@@ -139,6 +146,8 @@ all = [
"pymilvus",
# encoders
"sentence-transformers",
+ # graph-storages
+ "neo4j",
# retrievers
"rank-bm25",
]
diff --git a/test/storages/graph_storages/test_neo4j_graph.py b/test/storages/graph_storages/test_neo4j_graph.py
new file mode 100644
index 000000000..c698291e8
--- /dev/null
+++ b/test/storages/graph_storages/test_neo4j_graph.py
@@ -0,0 +1,261 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from unstructured.documents.elements import Element
+
+from camel.storages import Neo4jGraph
+from camel.storages.graph_storages.graph_element import (
+ GraphElement,
+ Node,
+ Relationship,
+)
+from camel.storages.graph_storages.neo4j_graph import (
+ BASE_ENTITY_LABEL,
+ NODE_PROPERTY_QUERY,
+ REL_PROPERTY_QUERY,
+ REL_QUERY,
+)
+
+test_data = [
+ GraphElement(
+ nodes=[
+ Node(id="id_subj", type="type_subj"),
+ Node(id="id_obj", type="type_obj"),
+ ],
+ relationships=[
+ Relationship(
+ subj=Node(id="id_subj", type="type_subj"),
+ obj=Node(id="id_obj", type="type_obj"),
+ type="type_rel",
+ )
+ ],
+ source=Element(element_id="a04b820b51c760a41415c57c1eef8f08"),
+ )
+]
+
+url = "neo4j+s://5af77aab.databases.neo4j.io"
+username = "neo4j"
+password = "SEK_Fx5Bx-BkRwMx6__zM_TOPqXLWEP-czuIZ_u7-zE"
+
+
+def test_cypher_return_correct_schema() -> None:
+ r"""Test that chain returns direct results.
+ Tested graph.query and graph.refresh_schema.
+ """
+ graph = Neo4jGraph(
+ url=url,
+ username=username,
+ password=password,
+ )
+ # Delete all nodes in the graph
+ graph.query("MATCH (n) DETACH DELETE n")
+ # Create two nodes and a relationship
+ graph.query("""
+ CREATE (la:LabelA {property_a: 'a'})
+ CREATE (lb:LabelB)
+ CREATE (lc:LabelC)
+ MERGE (la)-[:REL_TYPE]-> (lb)
+ MERGE (la)-[:REL_TYPE {rel_prop: 'abc'}]-> (lc)
+ """)
+ # Refresh schema information
+ graph.refresh_schema()
+
+ node_properties = graph.query(
+ NODE_PROPERTY_QUERY, params={"EXCLUDED_LABELS": [BASE_ENTITY_LABEL]}
+ )
+ relationships_properties = graph.query(
+ REL_PROPERTY_QUERY, params={"EXCLUDED_LABELS": [BASE_ENTITY_LABEL]}
+ )
+ relationships = graph.query(
+ REL_QUERY, params={"EXCLUDED_LABELS": [BASE_ENTITY_LABEL]}
+ )
+
+ expected_node_properties = [
+ {
+ "output": {
+ "properties": [{"property": "property_a", "type": "STRING"}],
+ "labels": "LabelA",
+ }
+ }
+ ]
+ expected_relationships_properties = [
+ {
+ "output": {
+ "type": "REL_TYPE",
+ "properties": [{"property": "rel_prop", "type": "STRING"}],
+ }
+ }
+ ]
+ expected_relationships = [
+ {"output": {"start": "LabelA", "type": "REL_TYPE", "end": "LabelB"}},
+ {"output": {"start": "LabelA", "type": "REL_TYPE", "end": "LabelC"}},
+ ]
+
+ assert node_properties == expected_node_properties
+ assert relationships_properties == expected_relationships_properties
+ # Order is not guaranteed with Neo4j returns
+ assert (
+ sorted(relationships, key=lambda x: x["output"]["end"])
+ == expected_relationships
+ )
+
+
+def test_neo4j_timeout() -> None:
+ r"""Test that neo4j uses the timeout correctly."""
+ graph = Neo4jGraph(
+ url=url, username=username, password=password, timeout=0.1
+ )
+ try:
+ graph.query("UNWIND range(0,100000,1) AS i MERGE (:Foo {id:i})")
+ except Exception as e:
+ assert (
+ e.code # type: ignore[attr-defined]
+ == "Neo.ClientError.Transaction."
+ "TransactionTimedOutClientConfiguration"
+ )
+
+
+def test_neo4j_truncate_values() -> None:
+ r"""Test that neo4j uses the timeout correctly."""
+ graph = Neo4jGraph(
+ url=url, username=username, password=password, truncate=True
+ )
+ # Delete all nodes in the graph
+ graph.query("MATCH (n) DETACH DELETE n")
+ # Create two nodes and a relationship
+ graph.query("""
+ CREATE (la:LabelA {property_a: 'a'})
+ CREATE (lb:LabelB)
+ CREATE (lc:LabelC)
+ MERGE (la)-[:REL_TYPE]-> (lb)
+ MERGE (la)-[:REL_TYPE {rel_prop: 'abc'}]-> (lc)
+ """)
+ graph.refresh_schema()
+
+ output = graph.query("RETURN range(0,130,1) AS result")
+ assert output == [{}]
+
+
+def test_neo4j_add_data() -> None:
+ r"""Test that neo4j correctly import graph element."""
+ graph = Neo4jGraph(
+ url=url, username=username, password=password, truncate=True
+ )
+ # Delete all nodes in the graph
+ graph.query("MATCH (n) DETACH DELETE n")
+ # Remove all constraints
+ graph.query("CALL apoc.schema.assert({}, {})")
+ graph.refresh_schema()
+ # Create two nodes and a relationship
+ graph.add_graph_elements(test_data)
+ output = graph.query(
+ "MATCH (n) RETURN labels(n) AS label, count(*) AS count ORDER BY label"
+ )
+ assert output == [
+ {"label": ["type_obj"], "count": 1},
+ {"label": ["type_subj"], "count": 1},
+ ]
+ assert graph.structured_schema["metadata"]["constraint"] == []
+
+
+def test_neo4j_add_data_source() -> None:
+ r"""Test that neo4j correctly import graph element with source."""
+ graph = Neo4jGraph(
+ url=url, username=username, password=password, truncate=True
+ )
+ # Delete all nodes in the graph
+ graph.query("MATCH (n) DETACH DELETE n")
+ # Remove all constraints
+ graph.query("CALL apoc.schema.assert({}, {})")
+ graph.refresh_schema()
+ # Create two nodes and a relationship
+ graph.add_graph_elements(test_data, include_source=True)
+ output = graph.query(
+ "MATCH (n) RETURN labels(n) AS label, count(*) AS count ORDER BY label"
+ )
+ assert output == [
+ {"label": ["Element"], "count": 1},
+ {"label": ["type_obj"], "count": 1},
+ {"label": ["type_subj"], "count": 1},
+ ]
+ assert graph.structured_schema["metadata"]["constraint"] == []
+
+
+def test_neo4j_add_data_base() -> None:
+ r"""Test that neo4j correctly import graph element with base_entity."""
+ graph = Neo4jGraph(
+ url=url, username=username, password=password, truncate=True
+ )
+ # Delete all nodes in the graph
+ graph.query("MATCH (n) DETACH DELETE n")
+ # Remove all constraints
+ graph.query("CALL apoc.schema.assert({}, {})")
+ graph.refresh_schema()
+ # Create two nodes and a relationship
+ graph.add_graph_elements(test_data, base_entity_label=True)
+ output = graph.query(
+ "MATCH (n) RETURN apoc.coll.sort(labels(n)) AS label, "
+ "count(*) AS count ORDER BY label"
+ )
+ assert output == [
+ {"label": [BASE_ENTITY_LABEL, "type_obj"], "count": 1},
+ {"label": [BASE_ENTITY_LABEL, "type_subj"], "count": 1},
+ ]
+ assert graph.structured_schema["metadata"]["constraint"] != []
+
+
+def test_neo4j_add_data_base_source() -> None:
+ r"""Test that neo4j correctly import graph element with base_entity and
+ source."""
+ graph = Neo4jGraph(
+ url=url, username=username, password=password, truncate=True
+ )
+ # Delete all nodes in the graph
+ graph.query("MATCH (n) DETACH DELETE n")
+ # Remove all constraints
+ graph.query("CALL apoc.schema.assert({}, {})")
+ graph.refresh_schema()
+ # Create two nodes and a relationship
+ graph.add_graph_elements(
+ test_data, base_entity_label=True, include_source=True
+ )
+ output = graph.query(
+ "MATCH (n) RETURN apoc.coll.sort(labels(n)) AS label, "
+ "count(*) AS count ORDER BY label"
+ )
+ assert output == [
+ {"label": ["Element"], "count": 1},
+ {"label": [BASE_ENTITY_LABEL, "type_obj"], "count": 1},
+ {"label": [BASE_ENTITY_LABEL, "type_subj"], "count": 1},
+ ]
+ assert graph.structured_schema["metadata"]["constraint"] != []
+
+
+def test_neo4j_filtering_labels() -> None:
+ r"""Test that neo4j correctly filters excluded labels."""
+ graph = Neo4jGraph(
+ url=url, username=username, password=password, truncate=True
+ )
+ # Delete all nodes in the graph
+ graph.query("MATCH (n) DETACH DELETE n")
+ # Remove all constraints
+ graph.query("CALL apoc.schema.assert({}, {})")
+ graph.query(
+ "CREATE (:`Excluded_Label_A`)-[:`Excluded_Rel_A`]->"
+ "(:`Excluded_Label_B`)"
+ )
+ graph.refresh_schema()
+
+ # Assert both are empty
+ assert graph.structured_schema["node_props"] == {}
+ assert graph.structured_schema["relationships"] == []