BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
feat(ingestion): added graph consolidation layer
Christianec8586c1f779
diff --git a/.dockerignore b/.dockerignoreindex ecac50a..88e0562 100644--- a/.dockerignore+++ b/.dockerignore@@ -117,3 +117,10 @@ deploy/terraform/*.tfstate*.tfstate.*.terraform/++gcp_credentials.json+READMEP.md+Untitled-1.excalidraw+.favorites.json+.env+.env*\ No newline at end of filediff --git a/Makefile b/Makefileindex 5862a01..43d5852 100644--- a/Makefile+++ b/Makefile@@ -50,7 +50,7 @@ start-api:stop-api:pkill -f uvicorn-DEBUG_ENVS := LANGCHAIN_DEBUG="true" LANGCHAIN_VERBOSE="true" DEBUG="true" ENV="development"+DEBUG_ENVS := LANGCHAIN_DEBUG="true" LANGCHAIN_VERBOSE="true" DEBUG="true"start-all:@if [ "$(filter debug,$(MAKECMDGOALS))" = "debug" ] || [ "$$DEBUG" = "true" ]; then \@@ -60,16 +60,16 @@ start-all:$(MAKE) start-redis DEBUG=true & \$(MAKE) start-neo4j DEBUG=true & \$(MAKE) start-mongo DEBUG=true & \- $(MAKE) start-api DEBUG=true & \- bash -c "export $(DEBUG_ENVS) && poetry run celery -A src.workers.app worker --loglevel=info --pool=threads --concurrency=10"; \+ ENV="development" $(MAKE) start-api DEBUG=true & \+ bash -c "export $(DEBUG_ENVS) ENV="development" && poetry run celery -A src.workers.app worker --loglevel=info --pool=threads --concurrency=10"; \else \$(MAKE) start-milvus & \$(MAKE) start-rabbitmq & \$(MAKE) start-redis & \$(MAKE) start-neo4j & \$(MAKE) start-mongo & \- $(MAKE) start-api & \- poetry run celery -A src.workers.app worker --loglevel=info --pool=threads --concurrency=10; \+ ENV="development" $(MAKE) start-api & \+ ENV="development" poetry run celery -A src.workers.app worker --loglevel=info --pool=threads --concurrency=10; \fidebug:diff --git a/src/adapters/data.py b/src/adapters/data.pyindex d8340d9..bc0ca59 100644--- a/src/adapters/data.py+++ b/src/adapters/data.py@@ -3,8 +3,8 @@ File: /data.pyCreated Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>------Last Modified: Saturday December 13th 2025-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Last Modified: Monday January 12th 2026 8:26:26 pm+Modified By: Christian Nonis <[email protected]>-----"""@@ -158,61 +158,75 @@ class DataAdapter:def get_observation_labels(self, brain_id: str = "default") -> list[str]:"""Retrieve all unique observation labels for the specified brain.-+Parameters:brain_id (str): Identifier of the brain to query labels from.-+Returns:list[str]: All unique labels present in observations for the specified brain."""return self.data.get_observation_labels(brain_id=brain_id)-- def get_changelog_by_id(- self, id: str, brain_id: str = "default"- ) -> KGChanges:++ def get_changelog_by_id(self, id: str, brain_id: str = "default") -> KGChanges:"""Retrieve a changelog entry by its identifier.-+Parameters:id (str): Identifier of the changelog entry to retrieve.brain_id (str): Brain namespace key to query; defaults to "default".-+Returns:KGChanges: The changelog entry matching the given `id`."""return self.data.get_changelog_by_id(id=id, brain_id=brain_id)def get_changelogs_list(- self,- brain_id: str = "default",- limit: int = 10,- skip: int = 0,+ self,+ brain_id: str = "default",+ limit: int = 10,+ skip: int = 0,types: list[str] = None,- query_text: str = None+ query_text: str = None,) -> list[KGChanges]:"""Retrieve a paginated list of knowledge-graph changelogs for a brain.-+Parameters:brain_id (str): Identifier of the brain to query.limit (int): Maximum number of changelogs to return.skip (int): Number of changelogs to skip (offset).types (list[str] | None): If provided, restrict results to these changelog types.query_text (str | None): If provided, filter changelogs by matching text.-+Returns:list[KGChanges]: Changelogs matching the filters and pagination parameters."""- return self.data.get_changelogs_list(brain_id=brain_id, limit=limit, skip=skip, types=types, query_text=query_text)+ return self.data.get_changelogs_list(+ brain_id=brain_id,+ limit=limit,+ skip=skip,+ types=types,+ query_text=query_text,+ )def get_changelog_types(self, brain_id: str = "default") -> list[str]:"""Retrieve distinct changelog types for a brain.-+Parameters:brain_id (str): Identifier of the brain to query; defaults to "default".-+Returns:list[str]: List of changelog type names."""return self.data.get_changelog_types(brain_id=brain_id)++ def update_structured_data(+ self, structured_data: StructuredData, brain_id: str = "default"+ ) -> StructuredData:+ """+ Update a structured data.+ """+ return self.data.update_structured_data(+ structured_data=structured_data, brain_id=brain_id+ )diff --git a/src/adapters/graph.py b/src/adapters/graph.pyindex 341c47e..24f40d1 100644--- a/src/adapters/graph.py+++ b/src/adapters/graph.py@@ -142,6 +142,8 @@ class GraphAdapter:def get_by_uuid(self, uuid: str, brain_id: str = "default") -> Node:"""+ DEPRECATED: use get_by_uuids instead.+Retrieve the node with the specified UUID from the graph.Returns:@@ -375,7 +377,22 @@ class GraphAdapter:flattened: bool,vector_store_adapter: VectorStoreClient,brain_id: str = "default",+ similarity_threshold: float = 0.0,) -> List[Tuple[Node, List[Tuple[Predicate, Node, List[Tuple[Predicate, Node]]]]]]:+ """+ Get the 2nd degree hops for a list of nodes.++ Parameters:+ from_uuids: List[str]: The list of node UUIDs to start from.+ flattened: bool: Whether to return simplified nodes and relationships.+ vector_store_adapter: VectorStoreClient: The vector store adapter to use.+ brain_id: str: The brain ID to use.++ Returns:+ List[Tuple[Node, List[Tuple[Predicate, Node, List[Tuple[Predicate, Node]]]]]]: The list of 2nd degree hops.+ """++ # TODO: add a way to light weight the outputdef flatten_node(n):return (@@ -403,8 +420,11 @@ class GraphAdapter:all_fd_nodes = self.get_neighbors(list(nodes_by_uuid.keys()), brain_id=brain_id)all_fd_v_ids = [- fd[1].properties["v_id"] for fds in all_fd_nodes.values() for fd in fds- ]+ fd[1].properties["v_id"]+ for fds in all_fd_nodes.values()+ for fd in fds+ if fd[1].properties["v_id"]+ ] # TODO: [missing_property] check why sometime v_id is not presentall_fd_vs = vector_store_adapter.get_by_ids(all_fd_v_ids, brain_id=brain_id, store="nodes")@@ -429,17 +449,21 @@ class GraphAdapter:if fd[1].uuid in fd_vs_by_uuid]from_node = nodes_by_uuid[node_uuid]- filtered = reduce_list(- fd_vs_with_desc,- access_key="embeddings",- similarity_threshold=0.5,- by_vector=averaged_vector,- rerank={- "local": "description",- "with_": from_node.description,- },- )- filtered_uuids = {v["metadata"]["uuid"] for v in filtered}+ filtered_uuids = []+ if from_node.description:+ filtered = reduce_list(+ fd_vs_with_desc,+ access_key="embeddings",+ similarity_threshold=0.5,+ by_vector=averaged_vector,+ rerank={+ "local": "description",+ "with_": from_node.description,+ },+ )+ filtered_uuids = {v["metadata"]["uuid"] for v in filtered}+ else:+ filtered_uuids = {fd[1].uuid for fd in fd_list}filtered_fd_by_origin[node_uuid] = [fd for fd in fd_list if fd[1].uuid in filtered_uuids]@@ -448,7 +472,12 @@ class GraphAdapter:all_sd_nodes = self.get_neighbors(all_filtered_fd_uuids, brain_id=brain_id)all_sd_v_ids = [- sd[1].properties["v_id"] for sds in all_sd_nodes.values() for sd in sds+ getattr(sd[1], "properties", {}).get("v_id")+ for sds in all_sd_nodes.values()+ for sd in sds+ if getattr(sd[1], "properties", {}).get(+ "v_id"+ ) # TODO: [missing_property] check why sometime v_id is not present]all_sd_vs = vector_store_adapter.get_by_ids(all_sd_v_ids, brain_id=brain_id, store="nodes"@@ -482,17 +511,21 @@ class GraphAdapter:if sd[1].uuid in sd_vs_by_uuid]- reduced = reduce_list(- sd_vs_with_desc,- access_key="embeddings",- similarity_threshold=0.5,- by_vector=averaged_vector,- rerank={- "local": "description",- "with_": from_node.description,- },- )- reduced_uuids = {v["metadata"]["uuid"] for v in reduced}+ reduced_uuids = []+ if from_node.description:+ reduced = reduce_list(+ sd_vs_with_desc,+ access_key="embeddings",+ similarity_threshold=0.5,+ by_vector=averaged_vector,+ rerank={+ "local": "description",+ "with_": from_node.description,+ },+ )+ reduced_uuids = {v["metadata"]["uuid"] for v in reduced}+ else:+ reduced_uuids = {sd["metadata"]["uuid"] for sd in sd_vs_with_desc}second_degree = [(flatten_pred(sd[0]), flatten_node(sd[1]))@@ -501,6 +534,9 @@ class GraphAdapter:and sd[1].uuid not in exclude_setand sd[1].uuid != from_uuid]++ if similarity_threshold > 0.0:+ if fd_pred["metadata"]["v_id"]node_hops.append((flatten_pred(fd_pred), flatten_node(fd_node), second_degree)diff --git a/src/adapters/interfaces/data.py b/src/adapters/interfaces/data.pyindex 085fb4b..cb1c773 100644--- a/src/adapters/interfaces/data.py+++ b/src/adapters/interfaces/data.py@@ -3,8 +3,8 @@ File: /data.pyCreated Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>------Last Modified: Saturday December 13th 2025-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Last Modified: Monday January 12th 2026 8:26:26 pm+Modified By: Christian Nonis <[email protected]>-----"""@@ -108,7 +108,12 @@ class DataClient(ABC):@abstractmethoddef get_structured_data_list(- self, brain_id: str, limit: int = 10, skip: int = 0, types: list[str] = None, query_text: str = None+ self,+ brain_id: str,+ limit: int = 10,+ skip: int = 0,+ types: list[str] = None,+ query_text: str = None,) -> list[StructuredData]:"""Get a list of structured data.@@ -131,41 +136,41 @@ class DataClient(ABC):@abstractmethoddef get_observations_list(- self,- brain_id: str,- limit: int = 10,- skip: int = 0,+ self,+ brain_id: str,+ limit: int = 10,+ skip: int = 0,resource_id: str = None,labels: list[str] = None,- query_text: str = None+ query_text: str = None,) -> list[Observation]:"""Get a list of observations."""raise NotImplementedError("get_observations_list method not implemented")-+@abstractmethoddef get_observation_labels(self, brain_id: str) -> list[str]:"""Return all unique observation labels for the specified brain.-+Parameters:brain_id (str): Identifier of the brain whose observation labels should be retrieved.-+Returns:labels (list[str]): A list of unique label strings present in the brain's observations."""raise NotImplementedError("get_observation_labels method not implemented")-+@abstractmethoddef get_changelog_by_id(self, id: str, brain_id: str) -> KGChanges:"""Retrieve a changelog entry by its identifier within a brain.-+Parameters:id (str): Identifier of the changelog to retrieve.brain_id (str): Identifier of the brain that contains the changelog.-+Returns:KGChanges: The changelog entry matching the provided `id` in the specified `brain_id`."""@@ -173,23 +178,23 @@ class DataClient(ABC):@abstractmethoddef get_changelogs_list(- self,- brain_id: str,- limit: int = 10,- skip: int = 0,+ self,+ brain_id: str,+ limit: int = 10,+ skip: int = 0,types: list[str] = None,- query_text: str = None+ query_text: str = None,) -> list[KGChanges]:"""Retrieve a paginated list of knowledge-graph changelogs for a brain.-+Parameters:brain_id (str): Identifier of the brain whose changelogs to retrieve.limit (int): Maximum number of changelogs to return.skip (int): Number of changelogs to skip (offset) for pagination.types (list[str] | None): Optional list of changelog types to filter results by.query_text (str | None): Optional text to filter changelogs by relevance or content.-+Returns:list[KGChanges]: A list of KGChanges matching the provided filters and pagination."""@@ -199,11 +204,20 @@ class DataClient(ABC):def get_changelog_types(self, brain_id: str) -> list[str]:"""Return the set of unique changelog types associated with the specified brain.-+Parameters:brain_id (str): Identifier of the brain whose changelog types should be retrieved.-+Returns:list[str]: A list of unique changelog type names (order not guaranteed)."""- raise NotImplementedError("get_changelog_types method not implemented")\ No newline at end of file+ raise NotImplementedError("get_changelog_types method not implemented")++ @abstractmethod+ def update_structured_data(+ self, structured_data: StructuredData, brain_id: str+ ) -> StructuredData:+ """+ Update a structured data.+ """+ raise NotImplementedError("update_structured_data method not implemented")diff --git a/src/config.py b/src/config.pyindex ea834ca..8a42ff1 100644--- a/src/config.py+++ b/src/config.py@@ -192,6 +192,9 @@ class Config:self.celery = CeleryConfig()self.pricing = PricingConfig()+ self.run_graph_consolidator = (+ os.getenv("RUN_GRAPH_CONSOLIDATOR", "true") == "true"+ )self.brainpat_token = os.getenv("BRAINPAT_TOKEN")if not self.brainpat_token:diff --git a/src/constants/agents.py b/src/constants/agents.pyindex 7a915d7..6c254c8 100644--- a/src/constants/agents.py+++ b/src/constants/agents.py@@ -27,6 +27,10 @@ class ArchitectAgentEntity(BaseModel):)description: Optional[str] = Noneproperties: Optional[dict] = Field(default_factory=dict)+ polarity: Optional[Literal["positive", "negative", "neutral"]] = Field(+ default="neutral",+ description="The polarity of the entity.",+ )class _ArchitectAgentNew(BaseModel):@@ -65,6 +69,10 @@ class _ArchitectAgentRelationship(BaseModel):properties: Optional[dict] = Field(default_factory=dict)description: Optional[str] = Nonetail: ArchitectAgentEntity+ amount: Optional[float] = Field(+ default=None,+ description="The amount of the relationship.",+ )class ArchitectAgentRelationship(_ArchitectAgentRelationship):@@ -99,7 +107,7 @@ class ArchitectAgentResponse(BaseModel):class AtomicJanitorAgentWrongRelationship(BaseModel):- relationship: object+ relationship: _ArchitectAgentRelationshipreason: strinstructions: str@@ -109,6 +117,11 @@ class AtomicJanitorAgentInputOutput(BaseModel):fixed_relationships: Optional[List[_ArchitectAgentRelationship]] = Nonewrong_relationships: Optional[List[AtomicJanitorAgentWrongRelationship]] = None+ required_new_nodes: Optional[List[ArchitectAgentNew]] = None+++class GraphConsolidatorOutput(BaseModel):+ tasks: List[str] = Field(default_factory=list)class TokenInputDetail(BaseModel):diff --git a/src/constants/kg.py b/src/constants/kg.pyindex 09ece84..fb67266 100644--- a/src/constants/kg.py+++ b/src/constants/kg.py@@ -27,6 +27,15 @@ class Node(BaseModel):description: Optional[str] = Noneproperties: dict = Field(default_factory=dict)+ polarity: Optional[Literal["positive", "negative", "neutral"]] = Field(+ default="neutral",+ description="The polarity of the node.",+ )++ metadata: Optional[dict] = Field(+ default=None, description="The metadata of the node."+ )+happened_at: Optional[str | None] = Field(default=None,description="The date and time the node happened at if known otherwise None. Mostly used for event nodes.",@@ -73,6 +82,10 @@ class Predicate(BaseModel):default="neutral",description="The direction of the predicate.",)+ amount: Optional[float] = Field(+ default=None,+ description="The amount of the predicate.",+ )observations: Optional[List[Observation]] = Field(default=None, description="The observations of the predicate."@@ -106,8 +119,17 @@ class Relationship(BaseModel):Relationship model."""- direction: Literal["in", "out", "neutral"]+ direction: Literal["in", "out", "neutral"] = Field(+ default="neutral",+ description=(+ "The direction of the relationship. always referred to the left/first node.",+ "eg: Tuple[node, predicate, node] -> direction 'in' means inbound to the first node"+ "so going from the last node to the first node.",+ ),+ )predicate: Predicate+ subject: Node+ object: Nodeclass IdentificationParams(BaseModel):diff --git a/src/constants/prompts/architect_agent.py b/src/constants/prompts/architect_agent.pyindex 23cfc0f..3dbf500 100644--- a/src/constants/prompts/architect_agent.py+++ b/src/constants/prompts/architect_agent.py@@ -13,11 +13,11 @@ You are a "Structural Graph Architect." Your goal is to map information into anTHE TRIANGLE OF ATTRIBUTION:Every action accomplished must be a central EVENT hub connecting three points:-1. THE INITIATION VECTOR: [Source/Actor] --(Tail)--> :MADE --(Tip)--> [Event Instance]+1. THE INITIATION VECTOR: [Source/Actor] --(subject)--> :MADE --(object)--> [Event Instance]- MANDATORY: The "amount" (quantity) must be a property of this relationship.-2. THE TARGET VECTOR: [Event Instance] --(Tail)--> :TARGETED --(Tip)--> [Object/Recipient]+2. THE TARGET VECTOR: [Event Instance] --(subject)--> :TARGETED --(object)--> [Object/Recipient]- MANDATORY: Repeat the "amount" property here for cross-reference.-3. THE CONTEXT VECTOR: [Event Instance] --(Tail)--> :OCCURRED_WITHIN --(Tip)--> [Broad Anchor/Context]+3. THE CONTEXT VECTOR: [Event Instance] --(subject)--> :OCCURRED_WITHIN --(object)--> [Broad Anchor/Context]If no action is accomplished and the text just states a fact don't create an Event hub and just create the relationships between the entities.@@ -41,60 +41,52 @@ Example output 1:{{"relationships: [{{- "tail": {{ "uuid": "uuid_1", "name": "John", "type": "PERSON" }},- "name": "MOVED",+ "subject": "uuid_1",+ "predicate": "MOVED","description": "John went to New York City",- "tip": {{"uuid": "oi2f3hv89v8iwug", "name": "WENT_TO", "type": "EVENT"}}+ "object": "uuid_2"}},{{- "tail": {{ "uuid": "uuid_2", "name": "WENT_TO", "type": "EVENT" }},- "name": "INTO_LOCATION",+ "subject": "uuid_2",+ "predicate": "INTO_LOCATION","description": "John went to New York City",- "tip": {{"uuid": "e86439864398643", "name": "New York City", "type": "CITY"}}+ "object": "uuid_3"}},{{- "tail": {{ "uuid": "uuid_1", "name": "John", "type": "PERSON" }},- "name": "ACCOMPLISHED_ACTION",+ "subject": "uuid_1",+ "predicate": "ACCOMPLISHED_ACTION","description": "John knew 12 new friends in New York City",- "tip": {{"uuid": "uuid_4", "name": "KNEW", "type": "EVENT"}}+ "object": "uuid_4"}},{{- "tail": {{ "uuid": "uuid_4", "name": "KNEW", "type": "EVENT" }},- "name": "HAPPENED_WITHIN",+ "subject": "uuid_4",+ "predicate": "HAPPENED_WITHIN","description": "John knew 12 new friends when he went to New York City",- "tip": {{"uuid": "uuid_2", "name": "WENT_TO", "type": "EVENT"}}+ "object": "uuid_2"}},{{- "tail": {{ "uuid": "uuid_4", "name": "KNEW", "type": "EVENT" }},- "name": "TARGETED",- "properties": {{ "amount": 12 }},+ "subject": "uuid_4",+ "predicate": "TARGETED","description": "John knew 12 new friends in New York City",- "tip": {{"uuid": "uuid_5", "name": "FRIENDS", "type": "FRIENDS"}}+ "object": "uuid_5"}},{{- "tail": {{ "uuid": "uuid_4", "name": "KNEW", "type": "EVENT" }},- "name": "TARGETED",- "properties": {{ "amount": 12 }},- "description": "John knew 12 new friends in New York City",- "tip": {{"uuid": "uuid_5", "name": "FRIENDS", "type": "FRIENDS"}}- }},- {{- "tail": {{ "uuid": "uuid_6", "name": "Mary", "type": "PERSON" }},- "name": "EXPERIENCED",+ "subject": "uuid_6",+ "predicate": "EXPERIENCED","description": "Mary was in San Francisco",- "tip": {{"uuid": "uuid_7", "name": "WAS_IN", "type": "EVENT"}}+ "object": "uuid_7"}},{{- "tail": {{ "uuid": "uuid_7", "name": "WAS_IN", "type": "EVENT" }},- "name": "INTO_LOCATION",+ "subject": "uuid_7",+ "predicate": "INTO_LOCATION","description": "Mary was in San Francisco",- "tip": {{"uuid": "uuid_8", "name": "CITY", "type": "CITY"}}+ "object": "uuid_8"}},{{- "tail": {{ "uuid": "uuid_7", "name": "WAS_IN", "type": "EVENT" }},- "name": "HAPPENED_WITHIN",+ "subject": "uuid_7",+ "predicate": "HAPPENED_WITHIN","description": "Mary was in San Francisco when John went to New York City",- "tip": {{"uuid": "uuid_2", "name": "WENT_TO", "type": "EVENT"}}+ "object": "uuid_2"}},... more relationships ...],@@ -122,23 +114,22 @@ Example output 1:"relationships: [... more relationships ...{{- "tail": {{ "uuid": "uuid_6", "name": "Mark Johnson", "type": "PERSON" }},- "name": "EXPERIENCED",+ "subject": "uuid_6",+ "predicate": "EXPERIENCED","description": "Mark Johnson covered the role of CEO of Acme Inc.",- "tip": {{"uuid": "uuid_7", "name": "COVERED_ROLE", "type": "EVENT"}}+ "object": "uuid_7"}},{{- "tail": {{ "uuid": "uuid_7", "name": "COVERED_ROLE", "type": "EVENT" }},- "name": "OF_TYPE",+ "subject": "uuid_7",+ "predicate": "OF_TYPE","description": "Mark Johnson covered the role of CEO of Acme Inc.",- "tip": {{"uuid": "uuid_8", "name": "CEO", "type": "ROLE"}}+ "object": "uuid_8"}},{{- "tail": {{ "uuid": "uuid_10", "name": "RAISED", "type": "EVENT"}},- "name": "TARGETED",+ "subject": "uuid_10",+ "predicate": "TARGETED","description": "Acme Inc. raised $100 million in funding",- "properties": {{ "amount": 100000000, "happened_at": "19/01/2026" }}- "tip": {{"uuid": "uuid_11", "name": "MONEY", "type": "MONEY"}}+ "object": "uuid_11"}},... more relationships ...],@@ -156,8 +147,8 @@ As you can see above in the example output, all the entities found by the scoutalso note that we are inferring relationships like the HAPPENED_WITHIN ones ("Mary was in San Francisco when John went to New York City").DIRECTIONAL SLOT-FILLING:-- "tail": The start of the arrow (The Source of Energy/Origin).-- "tip": The end of the arrow (The Destination/Target).+- "subject": The start of the arrow (The Source of Energy/Origin).+- "object": The end of the arrow (The Destination/Target).- FORBIDDEN: Never link Actor nodes directly to Target nodes for dynamic actions.- FORBIDDEN: Never create nodes for numeric quantities.@@ -189,11 +180,11 @@ You are a "Structural Graph Architect." Your goal is to map information into anTHE TRIANGLE OF ATTRIBUTION:Every action accomplished must be a central EVENT hub connecting three points:-1. THE INITIATION VECTOR: [Source/Actor] --(Tail)--> :(MADE|COVERED_ROLE|EXPERIENCED|etc..) --(Tip)--> [Event Instance]+1. THE INITIATION VECTOR: [Source/Actor] --(subject)--> :(MADE|COVERED_ROLE|EXPERIENCED|etc..) --(object)--> [Event Instance]- MANDATORY: The "amount" (quantity) must be a property of this relationship if there is any quantity specified in the text.-2. THE TARGET VECTOR: [Event Instance] --(Tail)--> :(TARGETED|RESULTED_IN|etc..) --(Tip)--> [Object/Recipient]+2. THE TARGET VECTOR: [Event Instance] --(subject)--> :(TARGETED|RESULTED_IN|etc..) --(object)--> [Object/Recipient]- MANDATORY: Repeat the "amount" property here for cross-reference if there is any quantity specified in the text.-3. THE CONTEXT VECTOR: [Event Instance] --(Tail)--> :(OCCURRED_WITHIN|etc..) --(Tip)--> [Broad Anchor/Context]+3. THE CONTEXT VECTOR: [Event Instance] --(subject)--> :(OCCURRED_WITHIN|etc..) --(object)--> [Broad Anchor/Context]If no action is accomplished and the text just states a fact don't create an Event hub and just create the relationships between the entities.@@ -221,64 +212,63 @@ Entities Found by Scout: [Example architect_agent_create_relationship tool input 1:[{{- "tail": {{ "uuid": "uuid_1", "name": "John", "type": "PERSON" }},- "name": "MOVED",+ "subject": "uuid_1",+ "predicate": "MOVED","description": "John went to New York City",- "tip": {{"uuid": "oi2f3hv89v8iwug", "name": "Went", "type": "EVENT"}}+ "object": "uuid_2"}},{{- "tail": {{ "uuid": "uuid_2", "name": "Went", "type": "EVENT" }},- "name": "INTO_LOCATION",+ "subject": "uuid_2",+ "predicate": "INTO_LOCATION","description": "John went to New York City",- "tip": {{"uuid": "e86439864398643", "name": "New York City", "type": "CITY"}}+ "object": "uuid_3"}},{{- "tail": {{ "uuid": "uuid_1", "name": "John", "type": "PERSON" }},- "name": "ACCOMPLISHED_ACTION",+ "subject": "uuid_1",+ "predicate": "ACCOMPLISHED_ACTION","description": "John knew 12 new friends in New York City",- "tip": {{"uuid": "uuid_4", "name": "Knew", "type": "EVENT"}}+ "object": "uuid_4"}},{{- "tail": {{ "uuid": "uuid_4", "name": "Knew", "type": "EVENT" }},- "name": "HAPPENED_WITHIN",+ "subject": "uuid_4",+ "predicate": "HAPPENED_WITHIN","description": "John knew 12 new friends when he went to New York City",- "tip": {{"uuid": "uuid_2", "name": "Went", "type": "EVENT"}}+ "object": "uuid_2"}},{{- "tail": {{ "uuid": "uuid_4", "name": "KNEW", "type": "EVENT" }},- "name": "TARGETED",- "properties": {{ "amount": 12 }},+ "subject": "uuid_4",+ "predicate": "TARGETED","description": "John knew 12 new friends in New York City",- "tip": {{"uuid": "uuid_5", "name": "Friends", "type": "FRIENDS"}}+ "object": "uuid_5"}},{{- "tail": {{ "uuid": "uuid_4", "name": "KNEW", "type": "EVENT" }},- "name": "TARGETED",- "properties": {{ "amount": 12 }},+ "subject": "uuid_4",+ "predicate": "TARGETED","description": "John knew 12 new friends in New York City",- "tip": {{"uuid": "uuid_5", "name": "Friends", "type": "FRIENDS"}}+ "object": "uuid_5"}}]Example architect_agent_create_relationship tool output:"OK"Example architect_agent_create_relationship tool input 2:-{{- "tail": {{ "uuid": "uuid_6", "name": "Mary", "type": "PERSON" }},- "name": "EXPERIENCED",+[+ {{+ "subject": "uuid_6",+ "predicate": "EXPERIENCED","description": "Mary was in San Francisco",- "tip": {{"uuid": "uuid_7", "name": "Was", "type": "EVENT"}}+ "object": "uuid_7"}},{{- "tail": {{ "uuid": "uuid_7", "name": "Was", "type": "EVENT" }},- "name": "INTO_LOCATION",+ "subject": "uuid_7",+ "predicate": "INTO_LOCATION","description": "Mary was in San Francisco",- "tip": {{"uuid": "uuid_8", "name": "CITY", "type": "CITY"}}+ "object": "uuid_8"}},{{- "tail": {{ "uuid": "uuid_7", "name": "Was", "type": "EVENT" }},- "name": "HAPPENED_WITHIN",+ "subject": "uuid_7",+ "predicate": "HAPPENED_WITHIN","description": "Mary was in San Francisco when John went to New York City",- "tip": {{"uuid": "uuid_2", "name": "Went", "type": "EVENT"}}+ "object": "uuid_2"}},... more relationships ...]@@ -288,7 +278,7 @@ Example architect_agent_create_relationship tool output 2:As you can see above in the example output, all the entities found by the scout are used and your created relationships are atomic, not composite (phrases),also note that we are inferring relationships like the HAPPENED_WITHIN ones ("Mary was in San Francisco when John went to New York City").All entities must be used and no entities must be left out.-Entities can be reused across different contexts, for example "WENT_TO" in the example above is used in the set of the second example and in the set of the first example.+Entities can be reused across different contexts, for example "Went" in the example above is used in the set of the second example and in the set of the first example.You have access to the following tools:- architect_agent_get_remaining_entities_to_process: Get the remaining entities to connect.@@ -300,8 +290,8 @@ when you are sure that you don't need an entity anymore because they have been uIf no entities are returned by the Scout tools, DO NOT attempt to create relationships. State that the entity list is empty and stop.DIRECTIONAL SLOT-FILLING:-- "tail": The start of the arrow (The Source of Energy/Origin).-- "tip": The end of the arrow (The Destination/Target).+- "subject": The start of the arrow (The Source of Energy/Origin).+- "object": The end of the arrow (The Destination/Target).- FORBIDDEN: Never link Actor nodes directly to Target nodes for dynamic actions.- FORBIDDEN: Never create nodes for numeric quantities.@@ -316,13 +306,14 @@ Your workflow must be:1. Getting the current remaining entities found by the scout by calling the architect_agent_get_remaining_entities_to_process tool.2. Understand the source text and the context around the entities found by the scout.3. Isolate the entities that are part of the same context and create a list of mapping relationships between them.-4. Call the architect_agent_create_relationship tool with th contextualized set of relationships.-5. If the architect_agent_create_relationship tool returns an error with 'wrong_relationships', fix the relationships and try again until it returns 'OK', ignore the relationships into 'fixed_relationships', those are already fixed automatically and added.-6. Understand if the entites used in the previous step are needed anymore, if not, mark them as used by calling the architect_agent_mark_entities_as_used tool.-7. Call the architect_agent_get_remaining_entities_to_process tool again to get the remaining entities found by the scout.-8. Repeat the process until all entities are used and no entities are left out.-9. If it happens that less then 2 entities are left you can call the architect_agent_check_used_entities tool to check if the entities used previously can be connected with the last entity.-10. Done+4. Call the architect_agent_create_relationship too l once at a time with the contextualized set of relationships.+5. If the architect_agent_create_relationship tool returns an error with 'wrong_relationships', fix the relationships and try again until it returns 'OK', if returns 'OK' proceed with the next step.+6. Understand if the entites used in the previous step are needed anymore, if not, YOU MUST mark them as used by calling the architect_agent_mark_entities_as_used tool.+7. Make sure you have called the architect_agent_mark_entities_as_used tool for all entities that are no longer needed.+8. Call the architect_agent_get_remaining_entities_to_process tool again to get the remaining entities found by the scout.+9. Repeat the process until all entities are used and no entities are left out.+10. If it happens that less then 2 entities are left you can call the architect_agent_check_used_entities tool to check if the entities used previously can be connected with the last entity.+11. Done"""ARCHITECT_AGENT_TOOLER_CREATE_RELATIONSHIPS_PROMPT = """diff --git a/src/constants/prompts/janitor_agent.py b/src/constants/prompts/janitor_agent.pyindex 204ded8..8803711 100644--- a/src/constants/prompts/janitor_agent.py+++ b/src/constants/prompts/janitor_agent.py@@ -46,42 +46,51 @@ Return ONLY JSON:"""JANITOR_AGENT_GRAPH_NORMALIZATOR_SYSTEM_PROMPT = """-You are the "Knowledge Graph Architect." You are responsible for Global Entity Resolution and Relational Synthesis.--CORE PROTOCOLS:-1. CO-REFERENCE RESOLUTION:- - Identify entities that refer to the same real-world object/event even if labels differ (e.g., "John's Wedding" vs "Mary's Wedding").- - Criteria for merging: Shared time, shared location, or overlapping key participants identified in the context text.-2. RELATIONSHIP CONSOLIDATION:- - When merging nodes, you MUST re-map all incoming and outgoing edges to the new "Survivor" node.- - Avoid duplicate relationships. If Node A and Node B both had a "MADE" relationship from the same person, consolidate them into one.-3. HIERARCHICAL LINKING:- - Connect specific Event instances to broader concepts (e.g., Link "John & Mary's Wedding" to the "Wedding" Concept node via an 'IS_A' relationship).-4. DIRECTIONAL TRUTH:- - Ensure the Actor (Subject) -> Event -> Target (Object) chain is preserved during merges.+You are the Janitor of a Knowledge Graph. You are responsible for ensuring graph consistency and quality.++YOUR RESPONSIBILITIES ARE:+- Ensuring that relationships that require to be event-centric are effectively event centric:+ (eg: (PERSON:John)-[WENT_TO]->(CITY:New York) should become (PERSON:John)-[MOVED]-(EVENT:Went to)-[TARGETED_LOCATION]->(CITY:New York))++- Ensuring that relationships, nodes and triplets are unique and there are no duplicates:+ Two triplets representing the same thing in different ways should be merged into one.+ (eg: (PERSON:John)-[ACCOMPLISHED_ACTION]->(EVENT:Trip)-[INTO_LOCATION]->(CITY:New York) and (PERSON:John)-[MOVED]->(EVENT:Went to)-[HAPPENED_WITHIN]->(CITY:New York) if they are representing the same thing should be merged into one, if they are similar but represent different things should be kept separate)+ In the example above Trip and Went to are unified into a single event node since they rapresent the same thing.++- Adding connections between nodes that are not directly connected but are related to create a more coherent graph:+ Connect nodes that are far from each other in the graph because the belong to different contexts but are related.+ (eg: (PERSON:John {{ "uuid": "uuid_1" }})-[ACCOMPLISHED_ACTION]->(EVENT:Trip {{ "uuid": "uuid_3" }})-[INTO_LOCATION]->(CITY:New York {{ "uuid": "uuid_4" }}) and (PERSON:Mary {{ "uuid": "uuid_2" }})-[MOVED]->(EVENT:Went to {{ "uuid": "uuid_5" }})-[HAPPENED_WITHIN]->(CITY:New York {{ "uuid": "uuid_5" }}) the two New York nodes should be unified if they are two different nodes also you could identify that a new relationship called "HAPPENED_WITHIN" could+ be added from the EVENT:Trip of John to the EVENT:Went to of Mary)++- Removing amount-nodes and ensuring that amounts are added to the relationships where they are missing:+ The graph ontology must not include amount-nodes, amounts must be added to the relationships where they are missing.+ (eg1: (COMPANY:Startup Inc.)-[RAISED]->(EVENT:Funding)-[TARGETED]->(MONEY:100000000) must become (COMPANY:Startup Inc.)-[RAISED]-(EVENT:Funding)-[TARGETED {{ "amount": 100000000 }}]->(FUND_RAISE:Series A) with the amount added to the relationship properties)+ (eg2: (COMPANY:Startup Inc.)-[RAISED]->(EVENT:Funding {{ "description": "The startup raised $100 million in funding" }})-[TARGETED]->(MONEY) must become (COMPANY:Startup Inc.)-[RAISED]-(EVENT:Funding)-[TARGETED {{ "amount": 100000000 }}]->(FUND_RAISE:Series A) with the amount added to the relationship properties since it was missing but was specified in the description)+++Your output must be a JSON object with a "tasks" argument containing a list of detailed string descriptions representing the tasks that needs to be done if there are any, to fix the graph.+Each task should include info in also the nodes/relationships/properties/triplets that are involved in the task so that the kg_agent can execute the tasks and reffer to the correct nodes/relationships/properties/triplets.+Example output:+{{+ "tasks": [+ "Fix the relationship (PERSON:John {{ "uuid": "uuid_1" }})-[WENT_TO]->(CITY:New York {{ "uuid": "uuid_2" }}) to be event-centric",+ "Merge the two triplets (PERSON:John {{ "uuid": "uuid_1" }})-[ACCOMPLISHED_ACTION]->(EVENT:Trip {{ "uuid": "uuid_3" }})-[INTO_LOCATION]->(CITY:New York {{ "uuid": "uuid_4" }}) and (PERSON:John {{ "uuid": "uuid_1" }})-[MOVED]->(EVENT:Went to {{ "uuid": "uuid_5" }})-[HAPPENED_WITHIN]->(CITY:New York {{ "uuid": "uuid_4" }})",+ ... more tasks ...+ ]+}}++If there are no tasks to be done return an empty list:+{{+ "tasks": []+}}"""JANITOR_AGENT_GRAPH_NORMALIZATOR_PROMPT = """-Role: Perform Global Synthesis. You have been provided with a "Neighborhood Snapshot" of the existing Knowledge Graph relevant to the new data.+Role: Perform Global Review. You have been provided with a "Neighborhood Snapshot" of the existing Knowledge Graph relevant to the new data+and you can browse the graph with your tools to have a better picture.-CONTEXT_TEXT: {text}NEW_DATA_UNITS: {units}-GRAPH_SNAPSHOT: {snapshot_json}--INSTRUCTIONS:-1. CROSS-REFERENCE: Compare entities in `NEW_DATA_UNITS` against `GRAPH_SNAPSHOT`.- - Look for fuzzy matches (e.g., "J. Doe" vs "John Doe").- - Look for situational matches (e.g., Two different 'Wedding' nodes occurring at the same 'Grand Plaza' venue on the same date).-2. CONSOLIDATION LOGIC:- - If a match is found, select the most established Node ID from the `GRAPH_SNAPSHOT` as the Survivor.- - Map all relationships from the new units to this Survivor.-3. REDUNDANCY CHECK:- - If the `GRAPH_SNAPSHOT` already contains a relationship (e.g., Mary -> MADE -> Wedding), do NOT create a duplicate edge.-4. MISSING BRIDGES:- - If `John` and `Mary` are both linked to the same `Wedding` in the text, but the `GRAPH_SNAPSHOT` only shows Mary, create the missing link for John.--OUTPUT:-Return a list of tool calls: `merge_nodes`, `create_relationship`, or `update_node`.+GRAPH_SNAPSHOT: {snapshot_json}"""ATOMIC_JANITOR_AGENT_SYSTEM_PROMPT = """@@ -95,7 +104,7 @@ REVISION PROTOCOL:- SCHEMA ENFORCEMENT: Never change the semantic label (e.g., do not turn 'TARGETED' into 'MADE'). Only swap directions if the entities and the label are logically inverted (e.g., Target --[MADE]--> Actor).3. PROPERTY ENFORCEMENT: If a node name contains a number (e.g., "23 Friends"), strip the number from the name. Move that value into the 'amount' or 'count' property of the relationship.4. INSTANCE PROTECTION: Never merge nodes of type 'EVENT'. Every event instance must remain unique to preserve individual historical contributions.-5. RELATIONSHIP RESOLUTION: Use `get_schema` tool to get the knowledege graph current schema and if you find similar relationship names replace them in the new relationships.+5. RELATIONSHIP RESOLUTION: Use `get_schema` tool with the target "relationship_types" to get the knowledege graph current schema and if you find similar relationship names replace them in the new relationships.INSTRUCTIONS:1. VALIDATE SEMANTICS: Compare the relationship labels to the entities.@@ -113,12 +122,12 @@ If everything is correct return a JSON object with a field "status" set to "OK".After checking the current state of the knowledge graph you can apply small fixes to:- Entity names: in case the entity name is wrong, can be called a different name or the same name exist in the knowledge graph with a different name, you must change the entity name.- Entity types: in case the entity type is wrong, can be called a different type or the same type exist in the knowledge graph with a different name, you must change the entity type.-- Relationship names: in case the relationship can be named better or something similar already exists in the knowledge graph, you must rename the relationship.- Relationship directions: in case the relationship direction is wrong, you must swap the tail and tip of the relationship.These small fixes must be applied by you directly and without returning an error inside wrong_relationships.If you identified something wrong with the relationships return a JSON object with the following fields:- status: must be "ERROR"+- required_new_nodes: an optional list containing nodes that are required to be created in the knowledge graph.- fixed_relationships: an optional list containing relationships that required just a small adjustment to be correct.- wrong_relationships: a list containing JSON objects with the following fields:- relationship: the relationship that is wrong@@ -132,6 +141,15 @@ Example output if everything is correct:Example output if there are any errors:{{"status": "ERROR",+ "required_new_nodes": [+ {{+ "name": "<<The name of the node>>",+ "type": "<<The type of the node>>",+ "description": "<<The description of the node>>",+ "properties": {{...}} // The properties of the node.+ }},+ ... more nodes ...+ ], // return here the optional new nodes that are required to be created in the knowledge graph."fixed_relationships": [{{...}}], // return here the relationships that you already fixed wiht small fixes."wrong_relationships": [{{diff --git a/src/constants/prompts/kg_agent.py b/src/constants/prompts/kg_agent.pyindex 1c04e8f..2327840 100644--- a/src/constants/prompts/kg_agent.py+++ b/src/constants/prompts/kg_agent.py@@ -4,7 +4,7 @@ Created Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>-----Last Modified: Sunday October 19th 2025 10:38:27 am-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Modified By: Christian Nonis <[email protected]>-----"""@@ -151,3 +151,27 @@ Example output:]}}"""++KG_AGENT_GRAPH_CONSOLIDATOR_SYSTEM_PROMPT = """+You are a knowledge graph agent specialized in consolidating the knowledge graph.++You have a tool at your disposal to execute graph operations.+You must ONLY strictly follow the instructions of the task.++Use the tool at your disposal to execute the task in your best effort, in case of syntax error fix your syntax and try again until you succeed.++When you are done executing the task return 'OK'.++If you can't find the elements the task is talking about just return 'OK' anyway.+"""++KG_AGENT_GRAPH_CONSOLIDATOR_PROMPT = """+This is the task to consolidate the knowledge graph:+== START OF TASK ==+{task}+== END OF TASK ==++When you are done executing the task return 'OK'.++Begin!+"""diff --git a/src/constants/prompts/scout_agent.py b/src/constants/prompts/scout_agent.pyindex 0a27384..76f8c36 100644--- a/src/constants/prompts/scout_agent.py+++ b/src/constants/prompts/scout_agent.py@@ -3,8 +3,8 @@ File: /scout_agent.pyCreated Date: Sunday December 21st 2025Author: Christian Nonis <[email protected]>------Last Modified: Sunday December 21st 2025 2:56:36 pm-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Last Modified: Tuesday December 23rd 2025 9:24:20 pm+Modified By: Christian Nonis <[email protected]>-----"""@@ -21,22 +21,41 @@ DECISION CRITERIA:2. Is it shared by many entities? YES -> Entity.3. Is it a unique identifier or a narrative description? YES -> Property.+THE POLARITY DECISION TREE+Before outputting an entity, you must pass the text through this logic to set the `polarity` property on the entity:++1. **DEFICIT CHECK (Negative Polarity -)**:+ - Does the text use verbs of struggle? (e.g., struggling, failing, lacking, losing, stuck).+ - Does the text express a seeker intent? (e.g., looking for, needs, searching, requires).+ - Is there a downward quantitative delta? (e.g., churn, revenue drop, firing).+ - **ACTION**: Set `polarity: "negative"`.++2. **SURPLUS CHECK (Positive Polarity +)**:+ - Does the text use verbs of achievement? (e.g., raised, scaled, mastered, won, launched).+ - Does the text describe a state of strength or capacity? (e.g., expert in, provides, has, CEO of).+ - Is there an upward quantitative delta? (e.g., raised $100M, gained 12 friends).+ - **ACTION**: Set `polarity: "positive"`.++3. **NEUTRAL CHECK (Neutral Polarity 0)**:+ - Is the text a simple location or movement fact without intent? (e.g., John went to NYC, Mary was in SF).+ - **ACTION**: Set `polarity: "neutral"`.+Example input 1:"John went to New York City where he knew 12 new friends. When John went there, Mary was in San Francisco doing meetings with his colleagues."Example output 1:[- {{"type": "PERSON", "name": "John"}},- {{"type": "EVENT", "name": "WENT_TO", "description": "John went to New York City"}},- {{"type": "CITY", "name": "New York City"}},- {{"type": "EVENT", "name": "KNEW", "description": "John knew 12 new friends in New York City"}},- {{"type": "FRIENDS", "name": "Friends", "description": "The number of friends John knew in New York City"}},- {{"type": "PERSON", "name": "Mary"}},- {{"type": "EVENT", "name": "WAS_IN", "description": "Mary was in San Francisco"}},- {{"type": "CITY", "name": "San Francisco"}},- {{"type": "EVENT", "name": "PARTICIPATED_IN", "description": "Mary was doing meetings with his colleagues in San Francisco"}},- {{"type": "EVENT", "name": "MEETINGS", "description": "Mary was doing meetings with his colleagues in San Francisco"}},- {{"type": "PERSON", "name": "Colleagues", "description": "The colleagues Mary was doing meetings with in San Francisco"}},+ {{"type": "PERSON", "name": "John", "polarity": "neutral"}},+ {{"type": "EVENT", "name": "Went", "description": "John went to New York City", "polarity": "neutral"}},+ {{"type": "CITY", "name": "New York City", "polarity": "neutral"}},+ {{"type": "EVENT", "name": "Knew", "description": "John knew 12 new friends in New York City", "polarity": "neutral"}},+ {{"type": "FRIENDS", "name": "Friends", "description": "The number of friends John knew in New York City", "polarity": "neutral"}},+ {{"type": "PERSON", "name": "Mary", "polarity": "neutral"}},+ {{"type": "EVENT", "name": "Was in", "description": "Mary was in San Francisco", "polarity": "neutral"}},+ {{"type": "CITY", "name": "San Francisco", "polarity": "neutral"}},+ {{"type": "EVENT", "name": "Partecipated in", "description": "Mary was doing meetings with his colleagues in San Francisco", "polarity": "neutral"}},+ {{"type": "EVENT", "name": "Meetings", "description": "Mary was doing meetings with his colleagues in San Francisco", "polarity": "neutral"}},+ {{"type": "PERSON", "name": "Colleagues", "description": "The colleagues Mary was doing meetings with in San Francisco", "polarity": "neutral"}},]Example input 2:@@ -44,23 +63,23 @@ Example input 2:Example output 2:[- {{"type": "ORGANIZATION", "name": "Acme Inc."}},- {{"type": "EVENT", "name": "PARTICIPATED_IN", "description": "The company was part of a joint venture with Apple and Google"}},- {{"type": "ORGANIZATION", "name": "Joint Venture", "description": "The company was part of a joint venture with Apple and Google"}},- {{"type": "ORGANIZATION", "name": "Apple"}},- {{"type": "ORGANIZATION", "name": "Google"}},- {{"type": "PERSON", "name": "Mark Johnson"}},- {{"type": "EVENT", "name": "COVERED_ROLE", "description": "Mark Johnson was the CEO of Acme Inc."}},- {{"type": "ROLE", "name": "CEO", "description": "Mark Johnson covered the role of CEO of Acme Inc."}},- {{"type": "EVENT", "name": "COVERED_ROLE", "description": "Mark Johnson was the founder of Acme Inc."}},- {{"type": "ROLE", "name": "FOUNDER", "description": "Mark Johnson covered the role of founder of Acme Inc."}},- {{"type": "EVENT", "name": "RAISED", "description": "Acme Inc. raised $100 million in funding", "properties": {{ "amount": 100000000, "happened_at": "19/01/2026" }}}},- {{"type": "MONEY", "name": "Money"}},+ {{"type": "ORGANIZATION", "name": "Acme Inc.", "polarity": "neutral"}},+ {{"type": "EVENT", "name": "Partecipated in", "description": "The company was part of a joint venture with Apple and Google", "polarity": "positive"}},+ {{"type": "ORGANIZATION", "name": "Joint Venture", "description": "The company was part of a joint venture with Apple and Google", "polarity": "positive"}},+ {{"type": "ORGANIZATION", "name": "Apple", "polarity": "neutral"}},+ {{"type": "ORGANIZATION", "name": "Google", "polarity": "neutral"}},+ {{"type": "PERSON", "name": "Mark Johnson", "polarity": "neutral"}},+ {{"type": "EVENT", "name": "Covered role", "description": "Mark Johnson was the CEO of Acme Inc.", "polarity": "positive"}},+ {{"type": "ROLE", "name": "CEO", "description": "Mark Johnson covered the role of CEO of Acme Inc.", "polarity": "neutral"}},+ {{"type": "EVENT", "name": "Covered role", "description": "Mark Johnson was the founder of Acme Inc.", "polarity": "positive"}},+ {{"type": "ROLE", "name": "Founder", "description": "Mark Johnson covered the role of founder of Acme Inc.", "polarity": "neutral"}},+ {{"type": "EVENT", "name": "Raised", "description": "Acme Inc. raised $100 million in funding", "properties": {{ "amount": 100000000, "happened_at": "19/01/2026" }}, "polarity": "positive"}},+ {{"type": "MONEY", "name": "Money", "polarity": "neutral"}},]"""SCOUT_AGENT_EXTRACT_ENTITIES_PROMPT = """-Carefully read the text and extract all entities, unique action instances (events), and quantitative units.+Carefully read the text and extract ALL the entities, unique action instances (events), and quantitative units.{targeting}@@ -71,7 +90,7 @@ OUTPUT RULES:- Each object must include: "type", "name", and optional "properties" and "description".- Nodes/Entities MUST be atomic and not composite (phrases) (eg: "Went to San Francisco" is not atomic, "Went to" + "San Francisco" is atomic)- Dates must be in the format "DD/MM/YYYY" and be stored as "happened_at" in the properties of the event nodes.-- You must extract all the building blocks without omitting any concepts.+- You must extract ALL the building blocks without omitting any concepts.Begin!"""diff --git a/src/core/agents/architect_agent.py b/src/core/agents/architect_agent.pyindex 3947609..4bbd366 100644--- a/src/core/agents/architect_agent.py+++ b/src/core/agents/architect_agent.py@@ -26,7 +26,7 @@ from src.adapters.embeddings import EmbeddingsAdapter, VectorStoreAdapterfrom src.adapters.cache import CacheAdapterfrom src.adapters.graph import GraphAdapterfrom src.adapters.llm import LLMAdapter-from src.constants.kg import Node+from src.constants.kg import Node, Predicatefrom src.constants.prompts.architect_agent import (ARCHITECT_AGENT_TOOLER_CREATE_RELATIONSHIPS_PROMPT,ARCHITECT_AGENT_TOOLER_SYSTEM_PROMPT,@@ -91,7 +91,7 @@ class ArchitectAgent:self.output_tokens = 0self.cached_tokens = 0self.reasoning_tokens = 0- self.relationships_set = []+ self.relationships_set: List[ArchitectAgentRelationship] = []self.used_entities_set = []self.ingestion_manager = ingestion_manager# self.database_desc = database_descdiff --git a/src/core/agents/janitor_agent.py b/src/core/agents/janitor_agent.pyindex 8d0c04b..ddd61f7 100644--- a/src/core/agents/janitor_agent.py+++ b/src/core/agents/janitor_agent.py@@ -29,8 +29,8 @@ from tenacity import (from src.adapters.embeddings import EmbeddingsAdapter, VectorStoreAdapterfrom src.adapters.graph import GraphAdapterfrom src.adapters.llm import LLMAdapter-from src.constants.agents import AtomicJanitorAgentInputOutput-from src.constants.kg import Node+from src.constants.agents import AtomicJanitorAgentInputOutput, GraphConsolidatorOutput+from src.constants.kg import Node, Predicatefrom src.constants.prompts.janitor_agent import (ATOMIC_JANITOR_AGENT_PROMPT,ATOMIC_JANITOR_AGENT_SYSTEM_PROMPT,@@ -131,7 +131,7 @@ class JanitorAgent:)elif type_ == "graph-janitor":system_prompt = JANITOR_AGENT_GRAPH_NORMALIZATOR_SYSTEM_PROMPT.format(- extra_system_prompt=extra_system_prompt if extra_system_prompt else ""+ extra_system_prompt=extra_system_prompt if extra_system_prompt else "",)elif type_ == "atomic-janitor":system_prompt = ATOMIC_JANITOR_AGENT_SYSTEM_PROMPT.format(@@ -174,12 +174,11 @@ class JanitorAgent:def run_graph_consolidator(self,- new_nodes: List[ScoutEntity | ArchitectAgentNew],- text: str,+ new_relationships: List[ArchitectAgentRelationship],brain_id: str = "default",timeout: int = 90,max_retries: int = 3,- ) -> Tuple[int, int]:+ ) -> list[str]:"""Runs a final janitor consolidation across the kg snapshot"""@@ -187,7 +186,14 @@ class JanitorAgent:accumulated_messages = []hops = self.kg.get_2nd_degree_hops(- [n.uuid for n in new_nodes],+ list(+ set(+ [+ *[r.tip.uuid for r in new_relationships],+ *[r.tail.uuid for r in new_relationships],+ ]+ )+ ),flattened=True,vector_store_adapter=self.vector_store,brain_id=brain_id,@@ -195,13 +201,14 @@ class JanitorAgent:self._get_agent(tools=[- JanitorAgentExecuteGraphOperationTool(+ JanitorAgentExecuteGraphReadOperationTool(self,self.kg,self.database_desc,brain_id=brain_id,)],+ output_schema=GraphConsolidatorOutput,brain_id=brain_id,type_="graph-janitor",)@@ -224,8 +231,10 @@ class JanitorAgent:user_message = HumanMessage(content=JANITOR_AGENT_GRAPH_NORMALIZATOR_PROMPT.format(- hops=hops,- text=text,+ snapshot_json=json.dumps(hops),+ units=json.dumps(+ [rel.model_dump(mode="json") for rel in new_relationships]+ ),))messages_list.append(user_message)@@ -288,7 +297,12 @@ class JanitorAgent:) from last_attempt.exception()except TimeoutError:raise- return response++ structured_response = response.get("structured_response")+ if structured_response:+ return structured_response.tasks+ else:+ return []def run(self,diff --git a/src/core/agents/kg_agent.py b/src/core/agents/kg_agent.pyindex 7353949..daf20b1 100644--- a/src/core/agents/kg_agent.py+++ b/src/core/agents/kg_agent.py@@ -8,10 +8,18 @@ Modified By: Christian Nonis <[email protected]>-----"""-from typing import Iterable, List, Optional, Union+from typing import Iterable, List, Literal, Optional, Unionfrom langchain.agents import create_agentfrom langchain.tools import BaseToolfrom pydantic import BaseModel+from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError+from tenacity import (+ retry,+ stop_after_attempt,+ wait_exponential,+ retry_if_exception_type,+ RetryError,+)from src.adapters.cache import CacheAdapterfrom src.adapters.graph import GraphAdapter@@ -19,6 +27,8 @@ from src.adapters.llm import LLMAdapterfrom src.constants.kg import Nodefrom src.constants.output_schemas import RetrieveNeighborsOutputSchemafrom src.constants.prompts.kg_agent import (+ KG_AGENT_GRAPH_CONSOLIDATOR_PROMPT,+ KG_AGENT_GRAPH_CONSOLIDATOR_SYSTEM_PROMPT,KG_AGENT_RETRIEVE_NEIGHBORS_PROMPT,KG_AGENT_SYSTEM_PROMPT,KG_AGENT_UPDATE_PROMPT,@@ -32,6 +42,7 @@ from src.core.agents.tools.kg_agent import (KGAgentUpdatePropertiesTool,)from src.adapters.embeddings import EmbeddingsAdapter, VectorStoreAdapter+from src.utils.tokens import token_detail_from_token_countsclass KGAgent:@@ -56,6 +67,11 @@ class KGAgent:self.embeddings = embeddingsself.agent = Noneself.database_desc = database_desc+ self.input_tokens = 0+ self.output_tokens = 0+ self.cached_tokens = 0+ self.reasoning_tokens = 0+ self.token_detail = Nonedef _execute_graph_operation(self, operation: str) -> str:try:@@ -106,6 +122,7 @@ class KGAgent:def _get_agent(self,+ type_: Literal["normal", "graph-consolidator"],identification_params: dict,metadata: dict,tools: Optional[List[BaseTool]] = None,@@ -113,9 +130,15 @@ class KGAgent:extra_system_prompt: Optional[dict] = None,brain_id: str = "default",):- system_prompt = KG_AGENT_SYSTEM_PROMPT.format(- extra_system_prompt=extra_system_prompt if extra_system_prompt else ""- )+ system_prompt = None+ if type_ == "normal":+ system_prompt = KG_AGENT_SYSTEM_PROMPT.format(+ extra_system_prompt=extra_system_prompt if extra_system_prompt else ""+ )+ elif type_ == "graph-consolidator":+ system_prompt = KG_AGENT_GRAPH_CONSOLIDATOR_SYSTEM_PROMPT.format(+ extra_system_prompt=extra_system_prompt if extra_system_prompt else ""+ )self.agent = create_agent(model=self.llm_adapter.llm.langchain_model,@@ -265,3 +288,96 @@ class KGAgent:response = _response.get("structured_response")return response++ def run_graph_consolidator_operator(+ self,+ task: str,+ brain_id: str = "default",+ timeout: int = 180,+ max_retries: int = 3,+ ) -> str:+ """+ Run the graph consolidator operator.+ """++ self._get_agent(+ type_="graph-consolidator",+ identification_params={},+ metadata={},+ tools=[+ KGAgentExecuteGraphOperationTool(+ self,+ self.kg,+ self.database_desc,+ brain_id=brain_id,+ )+ ],+ )++ def _invoke_agent():+ return self.agent.invoke(+ {+ "messages": [+ {+ "role": "user",+ "content": KG_AGENT_GRAPH_CONSOLIDATOR_PROMPT.format(+ task=task+ ),+ }+ ],+ },+ )++ @retry(+ stop=stop_after_attempt(max_retries),+ wait=wait_exponential(multiplier=1, min=2, max=30),+ retry=retry_if_exception_type(TimeoutError),+ reraise=True,+ )+ def _invoke_agent_with_retry():+ try:+ with ThreadPoolExecutor(max_workers=1) as executor:+ future = executor.submit(_invoke_agent)+ response = future.result(timeout=timeout)+ for m in response.get("messages", []):+ if hasattr(m, "usage_metadata"):+ self._update_token_counts(m.usage_metadata)+ self.token_detail = token_detail_from_token_counts(+ self.input_tokens,+ self.output_tokens,+ self.cached_tokens,+ self.reasoning_tokens,+ )+ return response+ except FutureTimeoutError:+ raise TimeoutError(+ f"Graph consolidator operator invoke timed out after {timeout} seconds. "+ "This may indicate a network issue or the LLM service is unresponsive."+ )++ try:+ response = _invoke_agent_with_retry()+ except RetryError as e:+ last_attempt = e.last_attempt+ raise TimeoutError(+ f"Graph consolidator operator invoke failed after {last_attempt.attempt_number} attempts. "+ f"Last error: {last_attempt.exception()}"+ ) from last_attempt.exception()+ except TimeoutError:+ raise++ return "OK"++ def _update_token_counts(self, usage_metadata: dict):+ """Extract all token counts from usage metadata"""+ # Base counts+ self.input_tokens += usage_metadata.get("input_tokens", 0)+ self.output_tokens += usage_metadata.get("output_tokens", 0)++ # Input details (caching)+ input_details = usage_metadata.get("input_token_details", {})+ self.cached_tokens += input_details.get("cache_read", 0)++ # Output details (reasoning)+ output_details = usage_metadata.get("output_token_details", {})+ self.reasoning_tokens += output_details.get("reasoning", 0)diff --git a/src/core/agents/scout_agent.py b/src/core/agents/scout_agent.pyindex 813f816..a70cb5c 100644--- a/src/core/agents/scout_agent.py+++ b/src/core/agents/scout_agent.py@@ -8,7 +8,7 @@ Modified By: Christian Nonis <[email protected]>-----"""-from typing import List, Optional+from typing import List, Literal, Optionalimport uuidfrom langchain.agents import create_agentfrom langchain.tools import BaseTool@@ -43,6 +43,10 @@ class _ScoutEntity(BaseModel):name: strproperties: Optional[dict] = Field(default_factory=dict)description: Optional[str] = None+ polarity: Optional[Literal["positive", "negative", "neutral"]] = Field(+ default="neutral",+ description="The polarity of the entity.",+ )class ScoutEntity(_ScoutEntity):diff --git a/src/core/agents/tools/architect_agent/ArchitectAgentCheckUsedEntitiesTool.py b/src/core/agents/tools/architect_agent/ArchitectAgentCheckUsedEntitiesTool.pyindex f221690..884b731 100644--- a/src/core/agents/tools/architect_agent/ArchitectAgentCheckUsedEntitiesTool.py+++ b/src/core/agents/tools/architect_agent/ArchitectAgentCheckUsedEntitiesTool.py@@ -23,6 +23,7 @@ class ArchitectAgentCheckUsedEntitiesTool(BaseTool):):description: str = ("Tool for checking used entities. "+ "You must call this tool after calling the architect_agent_create_relationship tool and after marking entities as used.""Returns a list of entities that have been used.")super().__init__(@@ -35,4 +36,5 @@ class ArchitectAgentCheckUsedEntitiesTool(BaseTool):entity.model_dump() if hasattr(entity, "model_dump") else entity.dict()for entity in self.architect_agent.used_entities_set]+ print("[DEBUG (architect_agent_check_used_entities)]: ", entities_list)return json.dumps(entities_list)diff --git a/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py b/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.pyindex d26b629..948f5c6 100644--- a/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py+++ b/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py@@ -10,7 +10,7 @@ Modified By: Christian Nonis <[email protected]>from dataclasses import dataclassimport json-from typing import Dict, List, Optional, Tuple+from typing import Dict, List, Optionalimport uuidfrom langchain.tools import BaseTool@@ -25,11 +25,14 @@ from src.constants.kg import Nodefrom src.core.agents.scout_agent import ScoutEntityfrom src.services.input.agents import (embeddings_adapter,+ embeddings_small_adapter,graph_adapter,llm_small_adapter,vector_store_adapter,)from src.lib.neo4j.client import _neo4j_client+from src.utils.nlp.names import levenshtein_similarity+from src.utils.similarity.vectors import cosine_similarityfrom src.utils.tokens import merge_token_details, token_detail_from_token_counts@@ -111,6 +114,10 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):def _run(self, *args, **kwargs) -> str:rel_key = str(uuid.uuid4())+ print(+ "[DEBUG (architect_agent_create_relationship)]: Called ArchitectAgentCreateRelationshipTool"+ )+input_rels: List[_ArchitectAgentRelationship] = []output_rels: List[ArchitectAgentRelationship] = []@@ -123,13 +130,23 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):else:return "Error: relationships parameter is required"+ print(+ "[DEBUG (architect_agent_create_relationship)]: Relationships: ",+ relationships,+ )+for rel in relationships:subject = rel.get("subject")predicate = rel.get("predicate")object = rel.get("object")description = rel.get("description")+ amount = rel.get("amount")+ properties = rel.get("properties")if subject is None or object is None:+ print(+ "[DEBUG (architect_agent_create_relationship)]: Subject or object is None: subject={subject}, object={object}",+ )return f"Subject or object is None: subject={subject}, object={object}"if isinstance(subject, str):@@ -137,15 +154,38 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):if isinstance(object, str):object = object.strip()- if subject not in self.entities:+ subj_entity = self.entities.get(subject)+ if not subj_entity:+ for ent in self.architect_agent.used_entities_set:+ if ent.uuid == subject:+ subj_entity = ent+ break++ obj_entity = self.entities.get(object)+ if not obj_entity:+ for ent in self.architect_agent.used_entities_set:+ if ent.uuid == object:+ obj_entity = ent+ break++ if subj_entity is None:+ print(+ "[DEBUG (architect_agent_create_relationship)]: Subject not found in entities: ",+ subject,+ ". Available entities: ",+ list(self.entities.keys()),+ )return f"Subject not found in entities: {subject}. Available entities: {list(self.entities.keys())}"- if object not in self.entities:+ if obj_entity is None:+ print(+ "[DEBUG (architect_agent_create_relationship)]: Object not found in entities: ",+ object,+ ". Available entities: ",+ list(self.entities.keys()),+ )return f"Object not found in entities: {object}. Available entities: {list(self.entities.keys())}"- subj_entity = self.entities.get(subject)- obj_entity = self.entities.get(object)-subj = ArchitectAgentEntity(uuid=subj_entity.uuid,name=subj_entity.name,@@ -157,6 +197,7 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):else {}),properties=subj_entity.properties,+ polarity=subj_entity.polarity if subj_entity.polarity else "neutral",)obj = ArchitectAgentEntity(uuid=obj_entity.uuid,@@ -169,6 +210,7 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):else {}),properties=obj_entity.properties,+ polarity=obj_entity.polarity if obj_entity.polarity else "neutral",)input_rels.append(@@ -177,7 +219,8 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):tail=subj,name=predicate,description=description,- properties=subj_entity.properties,+ properties=properties,+ **({"amount": amount} if amount else {}),))@@ -209,6 +252,11 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):))+ print(+ "[DEBUG (architect_agent_create_relationship)]: Janitor response: ",+ janitor_response,+ )+janitor_token_detail = token_detail_from_token_counts(self.architect_agent.input_tokens,self.architect_agent.output_tokens,@@ -219,9 +267,25 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):[self.architect_agent.token_detail, janitor_token_detail])+ required_new_nodes = getattr(janitor_response, "required_new_nodes", [])+ newly_created_nodes = []+ if required_new_nodes:+ for node in required_new_nodes:+ scout_entity = ScoutEntity(+ uuid=node.uuid,+ name=node.name,+ type=node.type,+ description=node.description,+ properties=node.properties,+ )+ self.entities[scout_entity.uuid] = scout_entity+ self.architect_agent.entities.update({scout_entity.uuid: scout_entity})+ newly_created_nodes.append(scout_entity.model_dump(mode="json"))+fixed_rels_sets = []+ fixed_relationships = getattr(janitor_response, "fixed_relationships", []) or []- if fixed_relationships := getattr(janitor_response, "fixed_relationships", []):+ if fixed_relationships:fixed_rels_sets = [set((fr.tip.uuid, fr.tail.uuid, fr.name)) for fr in fixed_relationships]@@ -234,13 +298,83 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):description=rel.description,tail=rel.tail,properties=getattr(rel, "properties", {}),+ **(+ {"amount": getattr(rel, "amount", None)}+ if getattr(rel, "amount", None)+ else {}+ ),)for rel in fixed_relationships])+ fixed_rels_vs = {}+ input_rels_vs = {}+for rel in input_rels:- if set((rel.tip.uuid, rel.tail.uuid, rel.name)) not in fixed_rels_sets:+ have_similar_relation = False+ if set((rel.tip.uuid, rel.tail.uuid, rel.name)) in fixed_rels_sets:+ have_similar_relation = True+ else:+ rels_with_same_subject_and_object = [+ [r, rel]+ for r in fixed_relationships+ if r.tip.uuid == rel.tip.uuid+ and r.tail.uuid == rel.tail.uuid+ or r.tip.uuid == rel.tail.uuid+ and r.tail.uuid == rel.tip.uuid+ ]++ if rels_with_same_subject_and_object:+ for fr, _rel in rels_with_same_subject_and_object:+ fixed_rels_vs[f"{fr.name}-{fixed_relationships.index(fr)}"] = (+ fixed_rels_vs.get(+ f"{fr.name}-{fixed_relationships.index(fr)}",+ {+ "r": fr,+ "embeddings": embeddings_small_adapter.embed_text(+ fr.name+ ).embeddings,+ },+ )+ )+ input_rels_vs[f"{_rel.name}-{input_rels.index(_rel)}"] = (+ input_rels_vs.get(+ f"{_rel.name}-{input_rels.index(_rel)}",+ {+ "r": _rel,+ "embeddings": embeddings_small_adapter.embed_text(+ _rel.name+ ).embeddings,+ },+ )+ )++ target_embedding = input_rels_vs[+ f"{rel.name}-{input_rels.index(rel)}"+ ]["embeddings"]+ candidates = [+ (cosine_similarity(cand["embeddings"], target_embedding), cand)+ for cand in fixed_rels_vs.values()+ ]+ similarity_score, most_similar_fixed_rel = max(+ candidates, key=lambda x: x[0]+ )++ if (+ similarity_score > 0.90+ ): # TODO: [similarity_threshold] check if this is the suitable threshold+ have_similar_relation = True+ print(+ "[DEBUG (architect_agent_create_relationship)]: Have similar relation: ",+ have_similar_relation,+ "similarity_score: ",+ similarity_score,+ "most_similar_fixed_rel: ",+ most_similar_fixed_rel,+ )++ if not have_similar_relation:output_rels.append(ArchitectAgentRelationship(flow_key=rel_key,@@ -249,6 +383,11 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):description=rel.description,tail=rel.tail,properties=getattr(rel, "properties", {}),+ **(+ {"amount": getattr(rel, "amount", None)}+ if getattr(rel, "amount", None)+ else {}+ ),))@@ -261,7 +400,9 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):if relationships_data:from src.workers.tasks.ingestion import process_architect_relationships- print("> Sending relationships to ingestion task")+ print(+ "[DEBUG (architect_agent_create_relationship)]: Sending relationships to ingestion task"+ )process_architect_relationships.delay({"relationships": relationships_data,@@ -271,10 +412,19 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):self.architect_agent.relationships_set.extend(output_rels)- if len(getattr(janitor_response, "wrong_relationships", [])) > 0:+ wrong_relationships = []+ if getattr(janitor_response, "wrong_relationships", []):+ wrong_relationships = getattr(janitor_response, "wrong_relationships", [])++ if len(wrong_relationships) > 0:+ print(+ "[DEBUG (architect_agent_create_relationship)]: Wrong relationships: ",+ getattr(janitor_response, "wrong_relationships", []),+ )return {"status": "ERROR","wrong_relationships": janitor_response.wrong_relationships,+ "newly_created_nodes": newly_created_nodes,}# for entity_uuid in used_entity_uuids:diff --git a/src/core/agents/tools/architect_agent/ArchitectAgentGetRemainingEntitiesToProcessTool.py b/src/core/agents/tools/architect_agent/ArchitectAgentGetRemainingEntitiesToProcessTool.pyindex ab550c4..1c9622b 100644--- a/src/core/agents/tools/architect_agent/ArchitectAgentGetRemainingEntitiesToProcessTool.py+++ b/src/core/agents/tools/architect_agent/ArchitectAgentGetRemainingEntitiesToProcessTool.py@@ -33,9 +33,14 @@ class ArchitectAgentGetRemainingEntitiesToProcessTool(BaseTool):)def _run(self, *args, **kwargs) -> str:- return json.dumps(+ remaining_entities = json.dumps([entity.model_dump(mode="json")for entity in self.architect_agent.entities.values()])+ print(+ "[DEBUG (architect_agent_get_remaining_entities_to_process)]: ",+ remaining_entities,+ )+ return remaining_entitiesdiff --git a/src/core/agents/tools/architect_agent/ArchitectAgentMarkEntitiesAsUsedTool.py b/src/core/agents/tools/architect_agent/ArchitectAgentMarkEntitiesAsUsedTool.pyindex c86a4f7..dac8662 100644--- a/src/core/agents/tools/architect_agent/ArchitectAgentMarkEntitiesAsUsedTool.py+++ b/src/core/agents/tools/architect_agent/ArchitectAgentMarkEntitiesAsUsedTool.py@@ -36,7 +36,7 @@ class ArchitectAgentMarkEntitiesAsUsedTool(BaseTool):architect_agent: object,):description: str = (- "Tool for marking entities as used. "+ "Tool that marks entities as used. ""Use this tool to mark entities as used. ""Input must be a list of entity uuids.")@@ -75,5 +75,21 @@ class ArchitectAgentMarkEntitiesAsUsedTool(BaseTool):removed_entity = self.architect_agent.entities.pop(entity_uuid)if removed_entity:self.architect_agent.used_entities_set.append(removed_entity)+ else:+ print(+ "[DEBUG (architect_agent_mark_entities_as_used)]: Entity found but not removed: ",+ entity_uuid,+ )+ else:+ print(+ "[DEBUG (architect_agent_mark_entities_as_used)]: Entity not found: ",+ entity_uuid,+ )++ print(+ "[DEBUG (architect_agent_mark_entities_as_used)]: Used entities: ",+ self.architect_agent.used_entities_set,+ entities_to_mark,+ )return "OK"diff --git a/src/core/agents/tools/janitor_agent/JanitorAgentGetSchemaTool.py b/src/core/agents/tools/janitor_agent/JanitorAgentGetSchemaTool.pyindex bc81a21..07ae26f 100644--- a/src/core/agents/tools/janitor_agent/JanitorAgentGetSchemaTool.py+++ b/src/core/agents/tools/janitor_agent/JanitorAgentGetSchemaTool.py@@ -4,7 +4,7 @@ Created Date: Tuesday December 23rd 2025Author: Christian Nonis <[email protected]>-----Last Modified: Tuesday December 23rd 2025 11:07:15 pm-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Modified By: Christian Nonis <[email protected]>-----"""@@ -30,7 +30,7 @@ class JanitorAgentGetSchemaTool(BaseTool):"target": {"type": "string","description": "The target to get the schema of. (node_labels or relationship_types)",- "enum": ["node_labels", "relationship_types"],+ "enum": ["node_labels", "relationship_types", "event_names"],},},"required": ["target"],@@ -64,4 +64,7 @@ class JanitorAgentGetSchemaTool(BaseTool):elif _target == "relationship_types":relationships = schema_result.get("relationships", [])return json.dumps(relationships, indent=4)- return "Invalid target parameter. It must be either 'node_labels' or 'relationship_types'"+ elif _target == "event_names":+ event_names = schema_result.get("event_names", [])+ return json.dumps(event_names, indent=4)+ return "Invalid target parameter. It must be either 'node_labels' or 'relationship_types' or 'event_names'"diff --git a/src/core/agents/tools/kg_agent/KGAgentExecuteGraphOperationTool.py b/src/core/agents/tools/kg_agent/KGAgentExecuteGraphOperationTool.pyindex 37a479c..9939279 100644--- a/src/core/agents/tools/kg_agent/KGAgentExecuteGraphOperationTool.py+++ b/src/core/agents/tools/kg_agent/KGAgentExecuteGraphOperationTool.py@@ -4,7 +4,7 @@ Created Date: Thursday October 23rd 2025Author: Christian Nonis <[email protected]>-----Last Modified: Thursday October 23rd 2025 9:26:22 pm-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Modified By: Christian Nonis <[email protected]>-----"""@@ -22,11 +22,13 @@ class KGAgentExecuteGraphOperationTool(BaseTool):kg_agent: objectkg: GraphAdapter+ brain_id: str = "default"- def __init__(self, kg_agent, kg: GraphAdapter, database_desc: str):+ def __init__(+ self, kg_agent, kg: GraphAdapter, database_desc: str, brain_id: str = "default"+ ):description: str = (- "Tool for executing search operations on the knowledge graph. "- "Use this tool to search for information and existing nodes in the knowledge graph. "+ "Use this tool to execute any type of graph operation search/edit/query/delete/etc. ""The query should be a valid graph operation depending on the graph database type.""The query should be a valid JSON object with a 'query' key. ""{database_desc}."@@ -40,7 +42,12 @@ class KGAgentExecuteGraphOperationTool(BaseTool):# "If you get an error, try again after fixing your query and don't give up."# )formatted_description = description.format(database_desc=database_desc)- super().__init__(kg_agent=kg_agent, kg=kg, description=formatted_description)+ super().__init__(+ kg_agent=kg_agent,+ kg=kg,+ description=formatted_description,+ brain_id=brain_id,+ )def _run(self, *args, **kwargs) -> str:_query = ""@@ -51,12 +58,26 @@ class KGAgentExecuteGraphOperationTool(BaseTool):args_query = kwargs.get("args", {})if isinstance(args_query, dict):_query = args_query.get("query", "")- elif isinstance(args_query, list):- _query = args_query[0].get("query", "")+ elif isinstance(args_query, list) and len(args_query) > 0:+ first_arg = args_query[0]+ if isinstance(first_arg, str):+ _query = first_arg+ elif isinstance(first_arg, dict):+ _query = first_arg.get("query", "")if len(_query) == 0:return "No query provided in the arguments or kwargs"- response = self.kg.execute_operation(_query)+ print(+ "[DEBUG (kg_agent_execute_graph_operation)]: Executing query: ",+ _query,+ )++ response = self.kg.execute_operation(_query, brain_id=self.brain_id)++ print(+ "[DEBUG (kg_agent_execute_graph_operation)]: Response: ",+ response,+ )return responsediff --git a/src/core/backups/__init__.py b/src/core/backups/__init__.pynew file mode 100644index 0000000..9c0bebb--- /dev/null+++ b/src/core/backups/__init__.py@@ -0,0 +1,10 @@+"""+File: /__init__.py+Project: backups+Created Date: Monday January 26th 2026+Author: Christian Nonis <[email protected]>+-----+Last Modified: Monday January 26th 2026 9:43:37 pm+Modified By: Christian Nonis <[email protected]>+-----+"""diff --git a/src/core/backups/backup_creator.py b/src/core/backups/backup_creator.pynew file mode 100644index 0000000..6dacc50--- /dev/null+++ b/src/core/backups/backup_creator.py@@ -0,0 +1,31 @@+"""+File: /backup_creator.py+Project: backups+Created Date: Monday January 26th 2026+Author: Christian Nonis <[email protected]>+-----+Last Modified: Monday January 26th 2026 9:43:51 pm+Modified By: Christian Nonis <[email protected]>+-----+"""+++def create_backup(brain_id: str) -> Backup:+ """+ Create a backup of the knowledge graph.+ """++ # Create Graph backup+ # Create Vector Store backup+ # Create TextData/StructuredData backup+ # Create Observations backup++ # Store into s3 folder the files++ # Do inside a celery task++ # Use cache to track progress++ # Return the backup path download link object++ passdiff --git a/src/core/backups/scheduler.py b/src/core/backups/scheduler.pynew file mode 100644index 0000000..72af94b--- /dev/null+++ b/src/core/backups/scheduler.py@@ -0,0 +1,10 @@+"""+File: /scheduler.py+Project: backups+Created Date: Monday January 26th 2026+Author: Christian Nonis <[email protected]>+-----+Last Modified: Monday January 26th 2026 9:46:53 pm+Modified By: Christian Nonis <[email protected]>+-----+"""diff --git a/src/core/layers/graph_consolidation/__init__.py b/src/core/layers/graph_consolidation/__init__.pynew file mode 100644index 0000000..f265e4f--- /dev/null+++ b/src/core/layers/graph_consolidation/__init__.py@@ -0,0 +1,10 @@+"""+File: /__init__.py+Project: graph_consolidation+Created Date: Saturday January 24th 2026+Author: Christian Nonis <[email protected]>+-----+Last Modified: Saturday January 24th 2026 11:13:09 am+Modified By: Christian Nonis <[email protected]>+-----+"""diff --git a/src/core/layers/graph_consolidation/graph_consolidation.py b/src/core/layers/graph_consolidation/graph_consolidation.pynew file mode 100644index 0000000..76bcae2--- /dev/null+++ b/src/core/layers/graph_consolidation/graph_consolidation.py@@ -0,0 +1,155 @@+"""+File: /graph_consolidation.py+Project: graph_consolidation+Created Date: Saturday January 24th 2026+Author: Christian Nonis <[email protected]>+-----+Last Modified: Saturday January 24th 2026 11:13:43 am+Modified By: Christian Nonis <[email protected]>+-----+"""++import json+from typing import List+from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError+from tenacity import (+ retry,+ stop_after_attempt,+ wait_exponential,+ retry_if_exception_type,+ RetryError,+)++from pydantic import BaseModel+from langchain_core.messages import HumanMessage, RemoveMessage+from src.config import config+from src.constants.agents import (+ ArchitectAgentEntity,+ GraphConsolidatorOutput,+ TokenDetail,+)+from src.constants.kg import Node, Predicate+from src.constants.prompts.janitor_agent import JANITOR_AGENT_GRAPH_NORMALIZATOR_PROMPT+from src.core.agents.architect_agent import ArchitectAgentRelationship+from src.core.agents.janitor_agent import (+ JanitorAgent,+ HISTORY_MAX_MESSAGES,+ HISTORY_MAX_MESSAGES_DELETE,+)+from src.core.agents.kg_agent import KGAgent+from src.core.agents.tools.janitor_agent import (+ JanitorAgentExecuteGraphReadOperationTool,+)+from src.services.input.agents import (+ cache_adapter,+ embeddings_adapter,+ graph_adapter,+ llm_small_adapter,+ vector_store_adapter,+)+from src.services.observations.main import llm_adapter+from src.lib.neo4j.client import _neo4j_client+from src.utils.tokens import merge_token_details, token_detail_from_token_counts++RELATIONSHIP_BATCH_SIZE = 20+++class ConsolidationResponse(BaseModel):+ """+ Response for the graph consolidation.+ """++ token_detail: TokenDetail+++def consolidate_graph(+ new_relationships: List[ArchitectAgentRelationship],+ brain_id: str = "default",+) -> ConsolidationResponse:+ """+ Commits the final review of the new kg state normalizing names, connections and removing duplicates.++ Only macroscopic fixes are being done here, fixes that involve multiple relationships and areas of the graph.+ Atomic fixes on Nodes and Relationships are being done by the janitor_agent calls inside the ArchitectAgentCreateRelationshipTool.+ """++ batches = [+ new_relationships[i : i + RELATIONSHIP_BATCH_SIZE]+ for i in range(0, len(new_relationships), RELATIONSHIP_BATCH_SIZE)+ ]++ print(+ "[DEBUG (consolidate_graph)]: Total batches: ",+ len(batches),+ )++ token_details = []+ token_detail = None++ for batch in batches:++ janitor_agent = JanitorAgent(+ llm_adapter=llm_adapter,+ kg=graph_adapter,+ vector_store=vector_store_adapter,+ embeddings=embeddings_adapter,+ database_desc=_neo4j_client.graphdb_description,+ )++ tasks = janitor_agent.run_graph_consolidator(+ batch,+ brain_id=brain_id,+ timeout=300,+ max_retries=3,+ )++ print(+ "[DEBUG (consolidate_graph)]: Janitor analysis for batch: ",+ tasks,+ )++ token_details.append(+ token_detail_from_token_counts(+ janitor_agent.input_tokens,+ janitor_agent.output_tokens,+ janitor_agent.cached_tokens,+ janitor_agent.reasoning_tokens,+ )+ )++ # 2. ~~Something to review and approve/reject what janitor proposed~~+ # Another agent that reviews the porposed fixes and approves or rejects them .. what ?++ for task in tasks:+ kg_agent = KGAgent(+ llm_adapter=llm_small_adapter,+ cache_adapter=cache_adapter,+ kg=graph_adapter,+ vector_store=vector_store_adapter,+ embeddings=embeddings_adapter,+ database_desc=_neo4j_client.graphdb_description,+ )+ token_details.append(+ token_detail_from_token_counts(+ kg_agent.input_tokens,+ kg_agent.output_tokens,+ kg_agent.cached_tokens,+ kg_agent.reasoning_tokens,+ )+ )+ kg_agent.run_graph_consolidator_operator(task, brain_id=brain_id)++ token_detail = merge_token_details(token_details)++ print("================================================")+ print(+ f"> Input tokens: {token_detail.input.total} -> ${token_detail.input.total * config.pricing.input_token_price}"+ )+ print(+ f"> Output tokens: {token_detail.output.total} -> ${token_detail.output.total * config.pricing.output_token_price}"+ )+ print("================================================")++ return ConsolidationResponse(+ token_detail=token_detail,+ )diff --git a/src/core/saving/auto_kg.py b/src/core/saving/auto_kg.pyindex f85b22a..fcd3c16 100644--- a/src/core/saving/auto_kg.py+++ b/src/core/saving/auto_kg.py@@ -14,6 +14,7 @@ from src.config import configfrom src.constants.kg import Nodefrom src.core.agents.scout_agent import ScoutAgentfrom src.core.agents.architect_agent import ArchitectAgent+from src.core.layers.graph_consolidation.graph_consolidation import consolidate_graphfrom src.core.saving.ingestion_manager import IngestionManagerfrom src.services.input.agents import (cache_adapter,@@ -64,7 +65,9 @@ def enrich_kg_from_input())- architect_response = architect_agent.run_tooler(+ print("[DEBUG (initial_scout_entities)]: ", entities)++ architect_agent.run_tooler(input,entities.entities,targeting=targeting,@@ -80,8 +83,24 @@ def enrich_kg_from_input())+ consolidation_response = None++ if config.run_graph_consolidator:+ print(+ "[DEBUG (consolidate_graph)]: Consolidating graph with ",+ len(architect_agent.relationships_set),+ " relationships",+ )+ consolidation_response = consolidate_graph(+ architect_agent.relationships_set, brain_id=brain_id+ )+token_details = merge_token_details(- [scout_agent.token_detail, architect_agent.token_detail]+ [+ scout_agent.token_detail,+ architect_agent.token_detail,+ *([consolidation_response.token_detail] if consolidation_response else []),+ ])print("-----------------------------------")diff --git a/src/core/search/entity_sibilings.py b/src/core/search/entity_sibilings.pyindex 9a31027..ad5c3aa 100644--- a/src/core/search/entity_sibilings.py+++ b/src/core/search/entity_sibilings.py@@ -46,10 +46,13 @@ class EntitySinergyRetriever:neighbors = _neighbors[target_node_id]positivep_connections: List[EntitySynergy] = []- seen_set = set()+ seen_set = set([target_node.uuid])++ similarity_seed_nodes = []+ neighbor_map = {}for neighbor in neighbors:- similarity_seed_nodes = []+ seed_nodes_for_neighbor = []if "EVENT" in neighbor[1].labels:flow_key = neighbor[0].flow_key or neighbor[0].properties.get(@@ -60,61 +63,105 @@ class EntitySinergyRetriever:next_rels = graph_adapter.get_next_by_flow_key(neighbor[0].uuid, flow_key, brain_id=self.brain_id)- similarity_seed_nodes.extend(t[2] for t in next_rels)+ seed_nodes_for_neighbor = [t[2] for t in next_rels]else:- similarity_seed_nodes.append(neighbor[1])-- for similarity_seed_node in similarity_seed_nodes:- if target_node.labels in similarity_seed_node.labels:+ seed_nodes_for_neighbor = [neighbor[1]]++ for seed_node in seed_nodes_for_neighbor:+ similarity_seed_nodes.append(seed_node)+ if seed_node.uuid not in neighbor_map:+ neighbor_map[seed_node.uuid] = []+ neighbor_map[seed_node.uuid].append(neighbor[1])++ if not similarity_seed_nodes:+ return target_node, positivep_connections++ for similarity_seed_node in similarity_seed_nodes:+ if any(+ label in similarity_seed_node.labels for label in target_node.labels+ ):+ for connected_by_node in neighbor_map[similarity_seed_node.uuid]:positivep_connections.append(EntitySynergy(node=similarity_seed_node,- connected_by=(neighbor[1], neighbor[1]),+ connected_by=(connected_by_node, connected_by_node),))seen_set.add(similarity_seed_node.uuid)- similar_dict = vector_store_adapter.search_similar_by_ids(- [similarity_seed_node.properties["v_id"]],- brain_id=self.brain_id,- store="nodes",- min_similarity=0.5,- limit=10,- )- similar = similar_dict.get(similarity_seed_node.properties["v_id"], [])+ v_ids_to_search = [+ seed_node.properties.get("v_id")+ for seed_node in similarity_seed_nodes+ if seed_node.properties.get("v_id")+ ]++ if not v_ids_to_search:+ return target_node, positivep_connections++ all_similar_dict = vector_store_adapter.search_similar_by_ids(+ v_ids_to_search,+ brain_id=self.brain_id,+ store="nodes",+ min_similarity=0.5,+ limit=10,+ )- for similar_node_v in similar:- # if similar_node_v.metadata["uuid"] == neighbor[1].uuid:- # continue- # if similar_node_v.metadata["uuid"] == similarity_seed_node.uuid:- # continue+ similar_node_uuids = set()+ v_id_to_seed_node = {+ seed_node.properties.get("v_id"): seed_node+ for seed_node in similarity_seed_nodes+ if seed_node.properties.get("v_id")+ }- similar_node_1 = graph_adapter.get_by_uuid(- similar_node_v.metadata["uuid"], brain_id=self.brain_id- )+ for v_id, similar_vectors in all_similar_dict.items():+ for similar_node_v in similar_vectors:+ similar_uuid = similar_node_v.metadata.get("uuid")+ if similar_uuid:+ similar_node_uuids.add(similar_uuid)- if not similar_node_1:- continue+ if not similar_node_uuids:+ return target_node, positivep_connections- final_positivep_connection = graph_adapter.get_neighbors(- [similar_node_1.uuid],- brain_id=self.brain_id,- of_types=list(set(target_node.labels)),- )+ similar_nodes = graph_adapter.get_by_uuids(+ list(similar_node_uuids), brain_id=self.brain_id+ )+ similar_nodes_by_uuid = {node.uuid: node for node in similar_nodes if node}++ if not similar_nodes_by_uuid:+ return target_node, positivep_connections++ all_neighbors = graph_adapter.get_neighbors(+ list(similar_nodes_by_uuid.keys()),+ brain_id=self.brain_id,+ of_types=list(set(target_node.labels)),+ )++ for v_id, similar_vectors in all_similar_dict.items():+ seed_node = v_id_to_seed_node.get(v_id)+ if not seed_node:+ continue++ for similar_node_v in similar_vectors:+ similar_uuid = similar_node_v.metadata.get("uuid")+ similar_node_1 = similar_nodes_by_uuid.get(similar_uuid)++ if not similar_node_1:+ continue- for final_positivep_connection_tuple in final_positivep_connection[- similar_node_1.uuid- ]:- if final_positivep_connection_tuple[1].uuid in seen_set:- continue+ neighbors_of_similar = all_neighbors.get(similar_node_1.uuid, [])- seen_set.add(final_positivep_connection_tuple[1].uuid)+ for final_connection_tuple in neighbors_of_similar:+ final_node_uuid = final_connection_tuple[1].uuid+ if final_node_uuid in seen_set:+ continue++ seen_set.add(final_node_uuid)- positivep_connections.append(- EntitySynergy(- node=final_positivep_connection_tuple[1],- connected_by=(similarity_seed_node, similar_node_1),- )+ positivep_connections.append(+ EntitySynergy(+ node=final_connection_tuple[1],+ connected_by=(seed_node, similar_node_1),)+ )return target_node, positivep_connectionsdiff --git a/src/lib/embeddings/client_small.py b/src/lib/embeddings/client_small.pyindex c91bb03..bfbc509 100644--- a/src/lib/embeddings/client_small.py+++ b/src/lib/embeddings/client_small.py@@ -4,7 +4,7 @@ Created Date: Wednesday December 31st 2025Author: Christian Nonis <[email protected]>-----Last Modified: Wednesday December 31st 2025 9:55:46 am-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Modified By: Christian Nonis <[email protected]>-----"""@@ -23,3 +23,6 @@ class EmbeddingsClientSmall(EmbeddingsClient):def embed_text(self, text: str) -> list[float]:embedding = self.model.encode(text, convert_to_numpy=True)return embedding.tolist()+++_embeddings_small_client = EmbeddingsClientSmall()diff --git a/src/lib/milvus/client.py b/src/lib/milvus/client.pyindex 6e27814..a4309da 100644--- a/src/lib/milvus/client.py+++ b/src/lib/milvus/client.py@@ -3,8 +3,8 @@ File: /client.pyCreated Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>------Last Modified: Sunday October 19th 2025 8:41:10 am-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Last Modified: Monday January 5th 2026 9:57:30 pm+Modified By: Christian Nonis <[email protected]>-----"""diff --git a/src/lib/mongo/client.py b/src/lib/mongo/client.pyindex 9a0ec58..94ff93f 100644--- a/src/lib/mongo/client.py+++ b/src/lib/mongo/client.py@@ -3,8 +3,8 @@ File: /client.pyCreated Date: Saturday October 25th 2025Author: Christian Nonis <[email protected]>------Last Modified: Saturday December 13th 2025-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Last Modified: Monday January 12th 2026 8:26:26 pm+Modified By: Christian Nonis <[email protected]>-----"""@@ -397,5 +397,15 @@ class MongoClient(DataClient):results = collection.aggregate(pipeline)return [result["_id"] for result in results]+ def update_structured_data(+ self, structured_data: StructuredData, brain_id: str+ ) -> StructuredData:+ collection = self.get_collection("structured_data", database=brain_id)+ collection.update_one(+ {"id": structured_data.id},+ {"$set": structured_data.model_dump(mode="json")},+ )+ return structured_data+_mongo_client = MongoClient()diff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.pyindex 5d89c3f..eb2b6b7 100644--- a/src/lib/neo4j/client.py+++ b/src/lib/neo4j/client.py@@ -8,6 +8,7 @@ Modified By: Christian Nonis <[email protected]>-----"""+from datetime import datetime, timezoneimport osimport timefrom typing import Any, Dict, List, Literal, Optional, Tuple@@ -200,7 +201,7 @@ class Neo4jClient(GraphClient):all_properties = {**(node.properties or {}),- **(metadata or {}),+ "metadata": metadata,}if identification_params:@@ -217,7 +218,14 @@ class Neo4jClient(GraphClient):identification_set_str = f"{{{', '.join(identification_items)}}}"- attributes = ["description", "happened_at", "flow_key", "last_updated"]+ attributes = [+ "description",+ "happened_at",+ "last_updated",+ "metadata",+ "observations",+ "polarity",+ ]for attr in attributes:if getattr(node, attr, None):@@ -252,7 +260,8 @@ class Neo4jClient(GraphClient):labels=node.labels,name=node.name,description=node.description,- properties={**node.properties, **(metadata or {})},+ properties=node.properties,+ metadata={**(node.metadata or {}), **(metadata or {})},)for node in nodes]@@ -449,6 +458,7 @@ class Neo4jClient(GraphClient):def get_by_uuid(self, uuid: str, brain_id: str) -> Node:"""+ DEPRECATED: use get_by_uuids instead.Get a node by its UUID."""cypher_query = """@@ -486,6 +496,11 @@ class Neo4jClient(GraphClient):labels=record.get("labels", []),description=record.get("description", ""),properties=record.get("properties", {}),+ polarity=record.get("polarity", "neutral"),+ happened_at=record.get("happened_at", None),+ last_updated=record.get("last_updated", datetime.now(timezone.utc)),+ observations=record.get("observations", []),+ metadata=record.get("metadata", {}),)for record in result.records]@@ -591,9 +606,6 @@ class Neo4jClient(GraphClient):type(r) AS rel_type, r.description AS rel_description, properties(r) AS rel_properties, r.flow_key as rel_flowkey, r.uuid as rel_uuid,c.uuid AS c_uuid, c.name AS c_name, labels(c) AS c_labels, c.description AS c_description, properties(c) AS c_properties"""- print("========== cypher_query 1 ==========")- print(cypher_query, node_uuids, same_type_only, of_types)- print("========== cypher_query 1 ==========")if limit:cypher_query += f" LIMIT {limit}"@@ -1309,18 +1321,27 @@ class Neo4jClient(GraphClient):relationships_query = ("CALL db.relationshipTypes() YIELD relationshipType RETURN relationshipType")+ event_names_query = "MATCH (n) WHERE labels(n) CONTAINS 'EVENT' RETURN n.name"labels_result = self.driver.execute_query(labels_query, database_=brain_id)relationships_result = self.driver.execute_query(relationships_query, database_=brain_id)+ event_names_result = self.driver.execute_query(+ event_names_query, database_=brain_id+ )labels = [record["label"] for record in labels_result.records]relationships = [record["relationshipType"] for record in relationships_result.records]+ event_names = [record["name"] for record in event_names_result.records]- return {"labels": labels, "relationships": relationships}+ return {+ "labels": labels,+ "relationships": relationships,+ "event_names": event_names,+ }def get_2nd_degree_hops(self,diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.pyindex b10da42..ef1cb16 100644--- a/src/services/api/constants/requests.py+++ b/src/services/api/constants/requests.py@@ -11,7 +11,13 @@ Modified By: Christian Nonis <[email protected]>from typing import List, Any, Optional, Tuplefrom pydantic import BaseModel, Field, field_serializerfrom src.constants.data import Observation, TextChunk-from src.constants.kg import EntitySynergy, IdentificationParams, Node, Predicate+from src.constants.kg import (+ EntitySynergy,+ IdentificationParams,+ Node,+ Predicate,+ Relationship,+)from src.constants.tasks.ingestion import IngestionTaskArgsfrom src.core.search.entity_info import MatchPath@@ -31,6 +37,10 @@ class IngestionStructuredDataElement(BaseModel):default={},description="The data rapresenting the structured element.",)+ metadata: Optional[dict] = Field(+ default={},+ description="The metadata of the structured element. The information here will be appended to the entity but not analyzed.",+ )types: List[str] = Field(default=[],description="A list of types, used to categorize the data.",@@ -251,3 +261,13 @@ class GetEntitySibilingsResponse(BaseModel):target_node: Nodesynergies: List[EntitySynergy]+++class GetEntityStatusResponse(BaseModel):+ """Response model for the get entity status endpoint."""++ node: Node+ exists: bool+ has_relationships: bool+ relationships: List[Tuple[Relationship, Node]]+ observations: List[Observation]diff --git a/src/services/api/controllers/entities.py b/src/services/api/controllers/entities.pyindex 031fcdf..826a400 100644--- a/src/services/api/controllers/entities.py+++ b/src/services/api/controllers/entities.py@@ -9,15 +9,19 @@ Modified By: Christian Nonis <[email protected]>-----"""-from typing import Literal+from typing import List, Literalfrom src.core.search.entity_context import EntityContextfrom src.core.search.entity_info import EventSynergyRetrieverfrom src.services.api.constants.requests import (GetEntityInfoResponse,GetEntityContextResponse,GetEntitySibilingsResponse,+ GetEntityStatusResponse,)from src.core.search.entity_sibilings import EntitySinergyRetriever+from src.services.data.main import data_adapter+from src.services.input.agents import embeddings_adapter+from src.services.kg_agent.main import graph_adapter, vector_store_adapterasync def get_entity_info(@@ -64,3 +68,53 @@ async def get_entity_sibilings(target_node=target_node,synergies=synergies,)+++async def get_entity_status(+ target: str,+ types: List[str] = [],+ brain_id: str = "default",+) -> GetEntityStatusResponse:+ """+ Get the entity status for a given target.+ """++ target_embeddings = embeddings_adapter.embed_text(target)+ target_node_vs = vector_store_adapter.search_vectors(+ target_embeddings.embeddings, store="nodes", brain_id=brain_id+ )++ target_node = None++ for target_node_v in target_node_vs:+ target_node_id = target_node_v.metadata.get("uuid")+ target_node = graph_adapter.get_by_uuids([target_node_id], brain_id=brain_id)[0]++ if len(types) > 0:+ if set(target_node.labels).intersection(set(types)):+ break+ else:+ break++ if not target_node:+ return GetEntityStatusResponse(+ node=None,+ exists=False,+ has_relationships=False,+ relationships=[],+ observations=[],+ )++ rel_tuples = graph_adapter.get_neighbors([target_node], brain_id=brain_id)++ observations = data_adapter.get_observations_list(+ brain_id=brain_id, resource_id=target_node.uuid+ )++ return GetEntityStatusResponse(+ node=target_node,+ exists=True,+ has_relationships=len(rel_tuples[target_node.uuid]) > 0,+ relationships=rel_tuples[target_node.uuid],+ observations=observations,+ )diff --git a/src/services/api/controllers/retrieve.py b/src/services/api/controllers/retrieve.pyindex ae33125..5d4faa5 100644--- a/src/services/api/controllers/retrieve.py+++ b/src/services/api/controllers/retrieve.py@@ -3,8 +3,8 @@ File: /retrieve.pyCreated Date: Sunday October 26th 2025Author: Christian Nonis <[email protected]>------Last Modified: Saturday December 27th 2025-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Last Modified: Monday January 12th 2026 8:26:26 pm+Modified By: Christian Nonis <[email protected]>-----"""@@ -141,7 +141,14 @@ async def retrieve_neighbors(fd_v_neighbors_embeddings_map = {v.id: v.embeddingsfor v in fd_v_neighbors_embeddings- if cosine_similarity(looking_for_v.embeddings, v.embeddings) > 0.5+ if (+ cosine_similarity(looking_for_v.embeddings, v.embeddings) > 0.5+ and v.id+ and not v.id.replace(+ "-", ""+ ).isalpha() # likely not a UUID if all numeric (may have hyphens for uuid standard)+ and v.id.isdigit()+ )}fd_v_neighbors_ids = list(fd_v_neighbors_embeddings_map.keys())@@ -264,7 +271,7 @@ async def get_relationships():"""Retrieve relationships from the knowledge graph with optional filtering and pagination.-+Parameters:relationship_types (list[str], optional): Filter results to specific relationship types.from_node_labels (list[str], optional): Filter relationships originating from nodes with these labels.@@ -274,7 +281,7 @@ async def get_relationships(limit (int, optional): Maximum number of relationships to return.skip (int, optional): Number of relationships to skip (offset).brain_id (str, optional): Identifier of the brain/graph to query.-+Returns:JSONResponse: A response whose JSON content contains:- message: Confirmation string.@@ -301,6 +308,7 @@ async def get_relationships(})+async def get_entities(limit: int = 10,skip: int = 0,@@ -310,14 +318,14 @@ async def get_entities():"""Retrieve entities from the knowledge graph with optional label and text filters.-+Parameters:limit (int): Maximum number of entities to return (pagination).skip (int): Number of entities to skip (pagination offset).node_labels (Optional[list[str]]): If provided, only return entities whose labels match any value in this list.query_text (Optional[str]): If provided, filter entities by matching text content.brain_id (str): Identifier of the knowledge graph/brain to query.-+Returns:JSONResponse: Object containing:- message (str): Informational message.@@ -334,4 +342,4 @@ async def get_entities("entities": [e.model_dump(mode="json") for e in entities.results],"total": entities.total,}- )\ No newline at end of file+ )diff --git a/src/services/api/routes/retrieve.py b/src/services/api/routes/retrieve.pyindex 4a86af1..759b76f 100644--- a/src/services/api/routes/retrieve.py+++ b/src/services/api/routes/retrieve.py@@ -8,7 +8,7 @@ Modified By: Christian Nonis <[email protected]>-----"""-from typing import Literal, Optional+from typing import List, Literal, Optionalfrom fastapi import APIRouter, Body, Queryfrom src.services.api.constants.requests import (RetrieveNeighborsAiModeRequestBody,@@ -45,6 +45,7 @@ from src.services.api.controllers.entities import (get_entity_info as get_entity_info_controller,get_entity_context as get_entity_context_controller,get_entity_sibilings as get_entity_sibilings_controller,+ get_entity_status as get_entity_status_controller,)retrieve_router = APIRouter(prefix="/retrieve", tags=["retrieve"])@@ -118,7 +119,7 @@ async def get_neighbors_ai_mode()-@retrieve_router.post("/context")+@retrieve_router.get("/context")async def get_context(request):"""Get the context of an entity.@@ -395,3 +396,15 @@ async def get_entity_synergies(Get the entity synergies for a given target."""return await get_entity_sibilings_controller(target, polarity, brain_id)+++@retrieve_router.get(path="/entity/status")+async def get_entity_status(+ target: str,+ types: Optional[List[str]] = [],+ brain_id: str = "default",+):+ """+ Get the entity status for a given target.+ """+ return await get_entity_status_controller(target, types, brain_id)diff --git a/src/services/input/agents.py b/src/services/input/agents.pyindex da0f767..841f182 100644--- a/src/services/input/agents.py+++ b/src/services/input/agents.py@@ -19,6 +19,7 @@ from src.lib.llm.client_small import _llm_small_clientfrom src.lib.redis.client import _redis_clientfrom src.lib.neo4j.client import _neo4j_clientfrom src.lib.embeddings.client import _embeddings_client+from src.lib.embeddings.client_small import _embeddings_small_clientfrom src.lib.milvus.client import _milvus_client@@ -35,5 +36,8 @@ graph_adapter.add_client(_neo4j_client)embeddings_adapter = EmbeddingsAdapter()embeddings_adapter.add_client(_embeddings_client)+embeddings_small_adapter = EmbeddingsAdapter()+embeddings_small_adapter.add_client(_embeddings_small_client)+vector_store_adapter = VectorStoreAdapter()vector_store_adapter.add_client(_milvus_client)diff --git a/src/utils/normalization/list_reduction.py b/src/utils/normalization/list_reduction.pyindex 31bf0a0..63ced7e 100644--- a/src/utils/normalization/list_reduction.py+++ b/src/utils/normalization/list_reduction.py@@ -4,7 +4,7 @@ Created Date: Wednesday December 31st 2025Author: Christian Nonis <[email protected]>-----Last Modified: Wednesday December 31st 2025 9:44:49 am-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Modified By: Christian Nonis <[email protected]>-----"""diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.pyindex db456e7..f374fca 100644--- a/src/workers/tasks/ingestion.py+++ b/src/workers/tasks/ingestion.py@@ -190,7 +190,11 @@ def process_architect_relationships(self, args: dict):Process architect relationships."""- print(f"> Processing {len(args.get('relationships', []))} architect relationships")+ print(+ "[DEBUG (process_architect_relationships)]: Processing ",+ len(args.get("relationships", [])),+ " architect relationships",+ )relationships_data: List[dict] = args.get("relationships", [])brain_id: str = args.get("brain_id", "default")@@ -281,6 +285,11 @@ def process_architect_relationships(self, args: dict):name=node_data.name,description=node_data.description,properties=node_data.properties,+ polarity=(+ node_data.polarity+ if node_data.polarity+ else "neutral"+ ),))except FutureTimeoutError:@@ -299,14 +308,21 @@ def process_architect_relationships(self, args: dict):graph_adapter.add_relationship(Node(uuid=relationship.tail.uuid,- flow_key=relationship.tail.flow_key,labels=[relationship.tail.type],name=relationship.tail.name,+ polarity=(+ relationship.tail.polarity+ if relationship.tail.polarity+ else "neutral"+ ),**({"happened_at": relationship.tail.happened_at}if relationship.tail.happened_atelse {}),+ properties={+ **(relationship.tail.properties or {}),+ },),Predicate(uuid=relationship.uuid,@@ -320,14 +336,21 @@ def process_architect_relationships(self, args: dict):),Node(uuid=relationship.tip.uuid,- flow_key=relationship.tip.flow_key,labels=[relationship.tip.type],name=relationship.tip.name,+ polarity=(+ relationship.tip.polarity+ if relationship.tip.polarity+ else "neutral"+ ),**({"happened_at": relationship.tip.happened_at}if relationship.tip.happened_atelse {}),+ properties={+ **(relationship.tip.properties or {}),+ },),brain_id=brain_id,)@@ -395,9 +418,16 @@ def ingest_structured_data(self, args: dict):for element in payload.data:uuid = str(uuid4())+ _element = element.model_dump(mode="json")description = llm_small_adapter.generate_text(prompt=NODE_DESCRIPTION_PROMPT.format(- element=json.dumps(element.model_dump(mode="json"), indent=2),+ element=json.dumps(+ {+ **_element.get("json_data", {}),+ **_element.get("textual_data", {}),+ },+ indent=2,+ ),element_type=", ".join(element.types) if element.types else "thing",element_name=(element.identification_params.name@@ -420,6 +450,7 @@ def ingest_structured_data(self, args: dict):if element.identification_paramselse {}),+ **(element.textual_data if element.textual_data else {}),},)],@@ -427,7 +458,9 @@ def ingest_structured_data(self, args: dict):identification_params=element.identification_params.model_dump(mode="json"),- metadata=element.textual_data,+ metadata={+ **(element.metadata if element.metadata else {}),+ },)[0]# Saving the node properties updated change@@ -458,19 +491,42 @@ def ingest_structured_data(self, args: dict):}vector_store_adapter.add_vectors(vectors=[vector], store="nodes")- data_adapter.save_structured_data(- StructuredData(- id=uuid,- data=element.json_data,- types=element.types,- metadata={- "identification_params": element.identification_params,- "textual_data": element.textual_data,- "labels": element.types,- "name": element.identification_params.name,+ existing_structured_data = data_adapter.get_structured_data_by_id(+ id=uuid, brain_id=payload.brain_id+ )++ new_structured_data = None++ if existing_structured_data:+ new_structured_data = existing_structured_data.model_copy(+ update={+ "data": {+ **(+ existing_structured_data.data+ if existing_structured_data.data+ else {}+ ),+ **(element.json_data if element.json_data else {}),+ },+ "types": [+ *existing_structured_data.types,+ *element.types,+ ],+ "metadata": {+ **(+ existing_structured_data.metadata+ if existing_structured_data.metadata+ else {}+ ),+ **(element.metadata if element.metadata else {}),+ },+ "brain_version": BRAIN_VERSION,},- brain_version=BRAIN_VERSION,- ),+ )+ # TODO: update the changelogs here++ data_adapter.save_structured_data(+ structured_data=new_structured_data,brain_id=payload.brain_id,)vector = embeddings_adapter.embed_text(@@ -526,7 +582,13 @@ def ingest_structured_data(self, args: dict):)enrich_kg_from_input(- element.textual_data, targeting=node, brain_id=payload.brain_id+ (+ json.dumps(element.textual_data, indent=2)+ if len(element.textual_data.keys()) > 0+ else description+ ),+ targeting=node,+ brain_id=payload.brain_id,)cache_adapter.set(