BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
fix(auto_kg): improved execution with external celery task for saving rels+entities
Christianb30b118339d0
diff --git a/src/adapters/graph.py b/src/adapters/graph.pyindex e800bf3..5874925 100644--- a/src/adapters/graph.py+++ b/src/adapters/graph.py@@ -3,8 +3,8 @@ File: /graph.pyCreated Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>------Last Modified: Saturday December 27th 2025-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Last Modified: Monday January 12th 2026 8:26:26 pm+Modified By: Christian Nonis <[email protected]>-----"""@@ -131,10 +131,10 @@ class GraphAdapter:def get_graph_relationships(self, brain_id: str = "default") -> list[str]:"""Retrieve the relationship types present in the graph.-+Parameters:brain_id (str): Identifier of the graph/brain to query.-+Returns:A list of relationship type names."""@@ -143,7 +143,7 @@ class GraphAdapter:def get_by_uuid(self, uuid: str, brain_id: str = "default") -> Node:"""Retrieve the node with the specified UUID from the graph.-+Returns:Node: The node matching the provided UUID."""@@ -283,28 +283,28 @@ class GraphAdapter:) -> Node | Predicate | None:"""Update properties on a graph node or relationship.-+Parameters:- uuid (str): UUID of the node or relationship to update.- updating (Literal["node", "relationship"]): Target entity type to update.- brain_id (str): Identifier of the graph (defaults to "default").- new_properties (dict): Properties to add or replace on the entity.- properties_to_remove (list[str]): Names of properties to remove from the entity.-+ uuid (str): UUID of the node or relationship to update.+ updating (Literal["node", "relationship"]): Target entity type to update.+ brain_id (str): Identifier of the graph (defaults to "default").+ new_properties (dict): Properties to add or replace on the entity.+ properties_to_remove (list[str]): Names of properties to remove from the entity.+Returns:- Node | Predicate | None: The updated entity, or `None` if the entity was not found.+ Node | Predicate | None: The updated entity, or `None` if the entity was not found."""return self.graph.update_properties(uuid, updating, brain_id, new_properties, properties_to_remove)-+def get_graph_relationship_types(self, brain_id: str = "default") -> list[str]:"""Retrieve all relationship type names present in the graph.-+Parameters:brain_id (str): Identifier of the graph (brain) to query. Defaults to "default".-+Returns:list[str]: List of unique relationship type names found in the graph."""@@ -313,21 +313,21 @@ class GraphAdapter:def get_graph_node_types(self, brain_id: str = "default") -> list[str]:"""Get all unique node types from the graph.-+@returns:A list of node type names available in the graph."""return self.graph.get_graph_node_types(brain_id)-+def get_graph_node_properties(self, brain_id: str = "default") -> list[str]:"""Retrieve all unique node property keys present in the graph.-+@returnslist[str]: A list of unique property key names present on nodes for the specified brain."""return self.graph.get_graph_node_properties(brain_id)-+def update_node(self,uuid: str,@@ -340,7 +340,7 @@ class GraphAdapter:) -> Node | None:"""Update an existing node's identifying fields, labels, and properties in the graph.-+Parameters:uuid (str): UUID of the node to update.brain_id (str): Identifier of the brain/graph where the node resides.@@ -349,7 +349,7 @@ class GraphAdapter:new_labels (Optional[list[str]]): New set of labels for the node; provide to replace the node's labels.new_properties (Optional[dict]): Properties to add or update on the node; keys are property names and values are their new values.properties_to_remove (Optional[list[str]]): List of property names to remove from the node.-+Returns:Node | None: The updated node if the update succeeded, or `None` if the node was not found."""@@ -510,4 +510,17 @@ class GraphAdapter:return hops-_graph_adapter = GraphAdapter()\ No newline at end of file+ def check_node_existence(+ self,+ uuid: str,+ name: str,+ labels: list[str],+ brain_id: str = "default",+ ) -> bool:+ """+ Check if a node exists in the graph.+ """+ return self.graph.check_node_existence(uuid, name, labels, brain_id)+++_graph_adapter = GraphAdapter()diff --git a/src/adapters/interfaces/graph.py b/src/adapters/interfaces/graph.pyindex 34da15f..d33238f 100644--- a/src/adapters/interfaces/graph.py+++ b/src/adapters/interfaces/graph.py@@ -3,8 +3,8 @@ File: /graph.pyCreated Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>------Last Modified: Saturday December 27th 2025-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Last Modified: Monday January 12th 2026 8:26:26 pm+Modified By: Christian Nonis <[email protected]>-----"""@@ -119,10 +119,10 @@ class GraphClient(ABC):def get_graph_relationships(self, brain_id: str) -> list[str]:"""Retrieve the relationship types present in the specified graph.-+Parameters:brain_id (str): Identifier of the brain/graph to query.-+Returns:list[str]: Relationship type names present in the graph."""@@ -136,11 +136,11 @@ class GraphClient(ABC):) -> Node:"""Retrieve a node identified by its UUID from the specified brain.-+Parameters:uuid (str): The node's UUID.brain_id (str): Identifier of the brain/graph to query.-+Returns:Node: The node matching the given UUID."""@@ -279,14 +279,14 @@ class GraphClient(ABC):) -> Node | Predicate | None:"""Update properties on a graph node or relationship.-+Parameters:uuid (str): UUID of the target node or relationship.updating (Literal["node", "relationship"]): Whether to update a node's properties or a relationship's properties.brain_id (str): Identifier of the graph/brain where the entity resides.new_properties (dict): Properties to set or overwrite on the target entity.properties_to_remove (list[str]): Property keys to remove from the target entity.-+Returns:Node | Predicate | None: The updated node or relationship object if the update succeeds, or `None` if no entity was found or no update was performed."""@@ -296,10 +296,10 @@ class GraphClient(ABC):def get_graph_relationship_types(self, brain_id: str) -> list[str]:"""List unique relationship types present in the specified graph.-+Parameters:brain_id (str): Identifier of the graph/brain to query.-+Returns:relationship_types (list[str]): Relationship type names present in the graph."""@@ -309,10 +309,10 @@ class GraphClient(ABC):def get_graph_node_types(self, brain_id: str) -> list[str]:"""Return all unique node types (labels) present in the graph for the given brain.-+Parameters:brain_id (str): Identifier of the brain/graph to query.-+Returns:list[str]: A list of node type names (labels) present in the specified graph."""@@ -322,15 +322,15 @@ class GraphClient(ABC):def get_graph_node_properties(self, brain_id: str) -> list[str]:"""Retrieve all unique node property keys present in the graph for the given brain.-+Parameters:brain_id (str): Identifier of the graph/brain to query.-+Returns:list[str]: Unique property key names found on nodes in the specified graph."""raise NotImplementedError("get_graph_node_properties method not implemented")-+@abstractmethoddef update_node(self,@@ -344,7 +344,7 @@ class GraphClient(ABC):) -> Node | None:"""Update a node's identifying fields, labels, and properties in the graph.-+Parameters:uuid (str): UUID of the node to update.brain_id (str): Identifier of the graph/brain where the node resides.@@ -353,7 +353,7 @@ class GraphClient(ABC):new_labels (Optional[list[str]]): New set of labels for the node; if provided, replaces existing labels.new_properties (Optional[dict]): Properties to add or update on the node; keys with values will be set or overwritten.properties_to_remove (Optional[list[str]]): List of property keys to remove from the node.-+Returns:Node | None: The updated Node if the node was found and modified, `None` if no matching node exists."""@@ -387,3 +387,16 @@ class GraphClient(ABC):The relationships are the 2nd degree hops."""raise NotImplementedError("get_2nd_degree_hops not implemented")++ @abstractmethod+ def check_node_existence(+ self,+ uuid: str,+ name: str,+ labels: list[str],+ brain_id: str,+ ) -> bool:+ """+ Check if a node exists in the graph.+ """+ raise NotImplementedError("check_node_existence method not implemented")diff --git a/src/constants/agents.py b/src/constants/agents.pyindex af7822a..9d2bbb3 100644--- a/src/constants/agents.py+++ b/src/constants/agents.py@@ -29,6 +29,8 @@ class ArchitectAgentEntity(BaseModel):default=None,description="The date and time the entity happened at if known otherwise None. Mostly used for event entities.",)+ description: Optional[str] = None+ properties: Optional[dict] = Field(default_factory=dict)class _ArchitectAgentNew(BaseModel):diff --git a/src/constants/kg.py b/src/constants/kg.pyindex 60d2f0d..a4d618a 100644--- a/src/constants/kg.py+++ b/src/constants/kg.py@@ -82,6 +82,8 @@ class Predicate(BaseModel):default=None, description="The observations of the predicate.")+ properties: dict = Field(default_factory=dict)+level: Optional[Literal["1", "2", "3"]] = Field(default=None,description=(diff --git a/src/constants/prompts/architect_agent.py b/src/constants/prompts/architect_agent.pyindex 3cf90f7..23cfc0f 100644--- a/src/constants/prompts/architect_agent.py+++ b/src/constants/prompts/architect_agent.py@@ -206,15 +206,15 @@ Example provided context:Entities Found by Scout: [{{"uuid": "uuid_1", "type": "PERSON", "name": "John"}},- {{"uuid": "uuid_2", "type": "EVENT", "name": "WENT_TO", "description": "John went to New York City"}},+ {{"uuid": "uuid_2", "type": "EVENT", "name": "Went", "description": "John went to New York City"}},{{"uuid": "uuid_3", "type": "CITY", "name": "New York City"}},- {{"uuid": "uuid_4", "type": "EVENT", "name": "KNEW", "description": "John knew 12 new friends in New York City"}},+ {{"uuid": "uuid_4", "type": "EVENT", "name": "Knew", "description": "John knew 12 new friends in New York City"}},{{"uuid": "uuid_5", "type": "UNIT", "name": "Friends", "description": "The number of friends John knew in New York City"}},{{"uuid": "uuid_6", "type": "PERSON", "name": "Mary"}},- {{"uuid": "uuid_7", "type": "EVENT", "name": "WAS_IN", "description": "Mary was in San Francisco"}},+ {{"uuid": "uuid_7", "type": "EVENT", "name": "Was", "description": "Mary was in San Francisco"}},{{"uuid": "uuid_8", "type": "CITY", "name": "San Francisco"}},- {{"uuid": "uuid_9", "type": "EVENT", "name": "PARTICIPATED_IN", "description": "Mary was doing meetings with his colleagues in San Francisco"}},- {{"uuid": "uuid_10", "type": "EVENT", "name": "MEETINGS", "description": "Mary was doing meetings with his colleagues in San Francisco"}},+ {{"uuid": "uuid_9", "type": "EVENT", "name": "Partecipation", "description": "Mary was doing meetings with his colleagues in San Francisco"}},+ {{"uuid": "uuid_10", "type": "EVENT", "name": "Meetings", "description": "Mary was doing meetings with his colleagues in San Francisco"}},{{"uuid": "uuid_11", "type": "PERSON", "name": "Colleagues", "description": "The colleagues Mary was doing meetings with in San Francisco"}},]@@ -224,10 +224,10 @@ Example architect_agent_create_relationship tool input 1:"tail": {{ "uuid": "uuid_1", "name": "John", "type": "PERSON" }},"name": "MOVED","description": "John went to New York City",- "tip": {{"uuid": "oi2f3hv89v8iwug", "name": "WENT_TO", "type": "EVENT"}}+ "tip": {{"uuid": "oi2f3hv89v8iwug", "name": "Went", "type": "EVENT"}}}},{{- "tail": {{ "uuid": "uuid_2", "name": "WENT_TO", "type": "EVENT" }},+ "tail": {{ "uuid": "uuid_2", "name": "Went", "type": "EVENT" }},"name": "INTO_LOCATION","description": "John went to New York City","tip": {{"uuid": "e86439864398643", "name": "New York City", "type": "CITY"}}@@ -236,27 +236,27 @@ Example architect_agent_create_relationship tool input 1:"tail": {{ "uuid": "uuid_1", "name": "John", "type": "PERSON" }},"name": "ACCOMPLISHED_ACTION","description": "John knew 12 new friends in New York City",- "tip": {{"uuid": "uuid_4", "name": "KNEW", "type": "EVENT"}}+ "tip": {{"uuid": "uuid_4", "name": "Knew", "type": "EVENT"}}}},{{- "tail": {{ "uuid": "uuid_4", "name": "KNEW", "type": "EVENT" }},+ "tail": {{ "uuid": "uuid_4", "name": "Knew", "type": "EVENT" }},"name": "HAPPENED_WITHIN","description": "John knew 12 new friends when he went to New York City",- "tip": {{"uuid": "uuid_2", "name": "WENT_TO", "type": "EVENT"}}+ "tip": {{"uuid": "uuid_2", "name": "Went", "type": "EVENT"}}}},{{"tail": {{ "uuid": "uuid_4", "name": "KNEW", "type": "EVENT" }},"name": "TARGETED","properties": {{ "amount": 12 }},"description": "John knew 12 new friends in New York City",- "tip": {{"uuid": "uuid_5", "name": "FRIENDS", "type": "FRIENDS"}}+ "tip": {{"uuid": "uuid_5", "name": "Friends", "type": "FRIENDS"}}}},{{"tail": {{ "uuid": "uuid_4", "name": "KNEW", "type": "EVENT" }},"name": "TARGETED","properties": {{ "amount": 12 }},"description": "John knew 12 new friends in New York City",- "tip": {{"uuid": "uuid_5", "name": "FRIENDS", "type": "FRIENDS"}}+ "tip": {{"uuid": "uuid_5", "name": "Friends", "type": "FRIENDS"}}}}]Example architect_agent_create_relationship tool output:@@ -266,19 +266,19 @@ Example architect_agent_create_relationship tool input 2:"tail": {{ "uuid": "uuid_6", "name": "Mary", "type": "PERSON" }},"name": "EXPERIENCED","description": "Mary was in San Francisco",- "tip": {{"uuid": "uuid_7", "name": "WAS_IN", "type": "EVENT"}}+ "tip": {{"uuid": "uuid_7", "name": "Was", "type": "EVENT"}}}},{{- "tail": {{ "uuid": "uuid_7", "name": "WAS_IN", "type": "EVENT" }},+ "tail": {{ "uuid": "uuid_7", "name": "Was", "type": "EVENT" }},"name": "INTO_LOCATION","description": "Mary was in San Francisco","tip": {{"uuid": "uuid_8", "name": "CITY", "type": "CITY"}}}},{{- "tail": {{ "uuid": "uuid_7", "name": "WAS_IN", "type": "EVENT" }},+ "tail": {{ "uuid": "uuid_7", "name": "Was", "type": "EVENT" }},"name": "HAPPENED_WITHIN","description": "Mary was in San Francisco when John went to New York City",- "tip": {{"uuid": "uuid_2", "name": "WENT_TO", "type": "EVENT"}}+ "tip": {{"uuid": "uuid_2", "name": "Went", "type": "EVENT"}}}},... more relationships ...]diff --git a/src/constants/prompts/janitor_agent.py b/src/constants/prompts/janitor_agent.pyindex e4c38dd..204ded8 100644--- a/src/constants/prompts/janitor_agent.py+++ b/src/constants/prompts/janitor_agent.py@@ -88,13 +88,14 @@ ATOMIC_JANITOR_AGENT_SYSTEM_PROMPT = """You are the "Knowledge Graph Janitor." You resolve entities, enforce directional logic, and preserve semantic intent.REVISION PROTOCOL:-1. IDENTITY RESOLUTION: Use `search_entities` for People, Places, Organizations, and Broad Contexts. If a match exists, replace UUID/Name with Database values.+1. IDENTITY RESOLUTION: Use `search_entities` to search into the knowledge graph if tips and tails already exist in the knowledge graph. If a match exists, replace UUID/Name with existing values.2. DIRECTIONAL AUDIT (RELATIONSHIP TYPES):- ACTOR-CENTRIC (e.g., MADE, INITIATED, PERFORMED): The 'tail' (source) must be the Subject (Person/Org) and the 'tip' (target) must be the EVENT.- IMPACT-CENTRIC (e.g., TARGETED, AFFECTED, RESULTED_IN): The 'tail' (source) must be the EVENT and the 'tip' (target) must be the Object/Recipient.- SCHEMA ENFORCEMENT: Never change the semantic label (e.g., do not turn 'TARGETED' into 'MADE'). Only swap directions if the entities and the label are logically inverted (e.g., Target --[MADE]--> Actor).3. PROPERTY ENFORCEMENT: If a node name contains a number (e.g., "23 Friends"), strip the number from the name. Move that value into the 'amount' or 'count' property of the relationship.4. INSTANCE PROTECTION: Never merge nodes of type 'EVENT'. Every event instance must remain unique to preserve individual historical contributions.+5. RELATIONSHIP RESOLUTION: Use `get_schema` tool to get the knowledege graph current schema and if you find similar relationship names replace them in the new relationships.INSTRUCTIONS:1. VALIDATE SEMANTICS: Compare the relationship labels to the entities.diff --git a/src/core/agents/architect_agent.py b/src/core/agents/architect_agent.pyindex 1468730..3947609 100644--- a/src/core/agents/architect_agent.py+++ b/src/core/agents/architect_agent.py@@ -52,6 +52,7 @@ from src.core.agents.tools.architect_agent.ArchitectAgentGetRemainingEntitiesToPfrom src.core.agents.tools.architect_agent.ArchitectAgentMarkEntitiesAsUsedTool import (ArchitectAgentMarkEntitiesAsUsedTool,)+from src.core.saving.ingestion_manager import IngestionManagerfrom src.utils.tokens import token_detail_from_token_counts# from src.core.agents.tools.kg_agent import (@@ -77,6 +78,7 @@ class ArchitectAgent:vector_store: VectorStoreAdapter,embeddings: EmbeddingsAdapter,# database_desc: str,+ ingestion_manager: IngestionManager,):self.llm_adapter = llm_adapterself.cache_adapter = cache_adapter@@ -91,10 +93,12 @@ class ArchitectAgent:self.reasoning_tokens = 0self.relationships_set = []self.used_entities_set = []+ self.ingestion_manager = ingestion_manager# self.database_desc = database_descdef _get_tools(self,+ text: Optional[str] = None,entities: Optional[Dict[str, ScoutEntity]] = None,brain_id: str = "default",targeting: Optional[Node] = None,@@ -103,6 +107,7 @@ class ArchitectAgent:return [ArchitectAgentCreateRelationshipTool(self,+ text=text,entities=entities,kg=self.kg,brain_id=brain_id,@@ -124,6 +129,7 @@ class ArchitectAgent:type_: Literal["single", "tooler"],tools: Optional[List[BaseTool]] = None,output_schema: Optional[BaseModel] = None,+ text: Optional[str] = None,extra_system_prompt: Optional[dict] = None,entities: Optional[Dict[str, ScoutEntity]] = None,brain_id: str = "default",@@ -145,7 +151,10 @@ class ArchitectAgent:toolsif toolselse self._get_tools(- entities=entities, brain_id=brain_id, targeting=targeting+ entities=entities,+ brain_id=brain_id,+ targeting=targeting,+ text=text,))if type_ == "tooler"@@ -387,6 +396,7 @@ class ArchitectAgent:self._get_agent(type_="tooler",+ text=text,brain_id=brain_id,entities=entities_dict,targeting=targeting,diff --git a/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py b/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.pyindex 5e0110c..ea5e496 100644--- a/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py+++ b/src/core/agents/tools/architect_agent/ArchitectAgentCreateRelationshipTool.py@@ -10,7 +10,7 @@ Modified By: Christian Nonis <[email protected]>from dataclasses import dataclassimport json-from typing import Dict, List, Optional+from typing import Dict, List, Optional, Tupleimport uuidfrom langchain.tools import BaseTool@@ -19,6 +19,7 @@ from src.constants.agents import (ArchitectAgentRelationship,ArchitectAgentEntity,AtomicJanitorAgentInputOutput,+ _ArchitectAgentRelationship,)from src.constants.kg import Nodefrom src.core.agents.scout_agent import ScoutEntity@@ -71,10 +72,12 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):entities: Dict[str, ScoutEntity]brain_id: str = "default"targeting: Optional[Node] = None+ text: strdef __init__(self,architect_agent: object,+ text: str,entities: Optional[Dict[str, ScoutEntity]],kg: GraphAdapter,brain_id: str = "default",@@ -95,6 +98,7 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):entities=entities_dict,brain_id=brain_id,targeting=targeting,+ text=text,)@dataclass@@ -107,7 +111,7 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):def _run(self, *args, **kwargs) -> str:rel_key = str(uuid.uuid4())- input_rels = []+ input_rels: List[_ArchitectAgentRelationship] = []output_rels: List[ArchitectAgentRelationship] = []if args and isinstance(args[0], dict) and "relationships" in args[0]:@@ -139,12 +143,6 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):if object not in self.entities:return f"Object not found in entities: {object}. Available entities: {list(self.entities.keys())}"- input_rels.append(- self._ArchitectAgentInputRelationship(- subject, predicate, object, description- )- )-subj_entity = self.entities.get(subject)obj_entity = self.entities.get(object)@@ -152,41 +150,45 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):uuid=subj_entity.uuid,name=subj_entity.name,type=subj_entity.type,- flow_key=rel_key,+ **({"flow_key": rel_key} if subj_entity.type != "EVENT" else {}),+ description=subj_entity.description,**({"happened_at": subj_entity.properties.get("happened_at")}if subj_entity.properties.get("happened_at")else {}),+ properties=subj_entity.properties,)obj = ArchitectAgentEntity(uuid=obj_entity.uuid,name=obj_entity.name,type=obj_entity.type,- flow_key=rel_key,+ **({"flow_key": rel_key} if obj_entity.type != "EVENT" else {}),+ description=obj_entity.description,**({"happened_at": obj_entity.properties.get("happened_at")}if obj_entity.properties.get("happened_at")else {}),+ properties=obj_entity.properties,)- output_rels.append(- ArchitectAgentRelationship(- flow_key=rel_key,+ input_rels.append(+ _ArchitectAgentRelationship(tip=obj,tail=subj,name=predicate,description=description,+ properties=subj_entity.properties,))- natural_lang = ", ".join(- [- f"""({rel.tail.name})-[:{rel.name} {{description: "{rel.description}"}}]->({rel.tip.name})"""- for rel in output_rels- ]- )+ # natural_lang = ", ".join(+ # [+ # f"""({rel.tail.name})-[:{rel.name} {{description: "{rel.description}"}}]->({rel.tip.name})"""+ # for rel in output_rels+ # ]+ # )from src.core.agents.janitor_agent import JanitorAgent@@ -198,13 +200,15 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):database_desc=_neo4j_client.graphdb_description,)- janitor_response = janitor_agent.run_atomic_janitor(- input_relationships=output_rels,- text=natural_lang,- targeting=self.targeting,- brain_id=self.brain_id,- timeout=300,- max_retries=3,+ janitor_response: AtomicJanitorAgentInputOutput = (+ janitor_agent.run_atomic_janitor(+ input_relationships=input_rels,+ text=self.text,+ targeting=self.targeting,+ brain_id=self.brain_id,+ timeout=300,+ max_retries=3,+ ))janitor_token_detail = token_detail_from_token_counts(@@ -217,7 +221,12 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):[self.architect_agent.token_detail, janitor_token_detail])+ fixed_rels_sets = []+if fixed_relationships := getattr(janitor_response, "fixed_relationships", []):+ fixed_rels_sets = [+ set((fr.tip.uuid, fr.tail.uuid, fr.name)) for fr in fixed_relationships+ ]output_rels.extend([ArchitectAgentRelationship(@@ -226,21 +235,56 @@ class ArchitectAgentCreateRelationshipTool(BaseTool):name=rel.name,description=rel.description,tail=rel.tail,+ properties=getattr(rel, "properties", {}),)for rel in fixed_relationships])+ for rel in input_rels:+ if set((rel.tip.uuid, rel.tail.uuid, rel.name)) not in fixed_rels_sets:+ output_rels.append(+ ArchitectAgentRelationship(+ flow_key=rel_key,+ tip=rel.tip,+ name=rel.name,+ description=rel.description,+ tail=rel.tail,+ properties=getattr(rel, "properties", {}),+ )+ )++ relationships_data = [+ rel.model_dump(mode="json")+ for rel in output_rels+ if isinstance(rel, ArchitectAgentRelationship)+ ]++ if relationships_data:+ from src.workers.tasks.ingestion import process_architect_relationships++ print("> Sending relationships to ingestion task")+ process_architect_relationships.delay(+ {+ "relationships": relationships_data,+ "brain_id": self.brain_id,+ }+ )+self.architect_agent.relationships_set.extend(output_rels)- from pprint import pprint- pprint(self.architect_agent.relationships_set)if len(getattr(janitor_response, "wrong_relationships", [])) > 0:- return janitor_response+ return {+ "status": "ERROR",+ "wrong_relationships": janitor_response.wrong_relationships,+ }# for entity_uuid in used_entity_uuids:# if entity_uuid in self.architect_agent.entities:# del self.architect_agent.entities[entity_uuid]# return natural_lang- return json.dumps({"status": "success", "insert_count": len(output_rels)})++ return json.dumps(+ {"status": "success", "message": "relationships created successfully"}+ )diff --git a/src/core/saving/auto_kg.py b/src/core/saving/auto_kg.pyindex 948cf04..f85b22a 100644--- a/src/core/saving/auto_kg.py+++ b/src/core/saving/auto_kg.py@@ -8,20 +8,13 @@ Modified By: Christian Nonis <[email protected]>-----"""-from concurrent.futures import (- Future,- ThreadPoolExecutor,- TimeoutError as FutureTimeoutError,-)-from typing import List, Optional, Tuple+from typing import Optional-from src.adapters.embeddings import EmbeddingsAdapter, VectorStoreAdapter-from src.adapters.graph import GraphAdapterfrom src.config import config-from src.constants.kg import Node, Predicate-from src.core.agents.janitor_agent import JanitorAgent-from src.core.agents.scout_agent import ScoutAgent, ScoutEntity-from src.core.agents.architect_agent import ArchitectAgent, ArchitectAgentRelationship+from src.constants.kg import Node+from src.core.agents.scout_agent import ScoutAgent+from src.core.agents.architect_agent import ArchitectAgent+from src.core.saving.ingestion_manager import IngestionManagerfrom src.services.input.agents import (cache_adapter,embeddings_adapter,@@ -29,75 +22,9 @@ from src.services.input.agents import (llm_small_adapter,vector_store_adapter,)-from src.lib.neo4j.client import _neo4j_clientfrom src.utils.tokens import merge_token_details, token_detail_from_token_counts-class IngestionManager:- def __init__(- self,- embeddings_adapter: EmbeddingsAdapter,- vector_store_adapter: VectorStoreAdapter,- graph_adapter: GraphAdapter,- ):- self.embeddings = embeddings_adapter- self.vector_store = vector_store_adapter- self.kg = graph_adapter- self.resolved_cache = {}- self.metadata = {}-- def process_node_vectors(self, node_data: ScoutEntity, brain_id):- if node_data.name in self.resolved_cache:- return node_data.uuid-- v_sub = self.embeddings.embed_text(node_data.name)- v_sub.metadata = {- "labels": [node_data.type],- "name": node_data.name,- "uuid": node_data.uuid,- }- if v_sub.embeddings:- v_ids = self.vector_store.add_vectors(- [v_sub], store="nodes", brain_id=brain_id- )- node_data.properties = {- **(node_data.properties or {}),- "v_id": v_ids[0],- }- self.resolved_cache[node_data.name] = v_ids[0]- return node_data.uuid- else:- print("[ ! ]Node not embedded:", node_data)- return node_data.uuid-- def process_rel_vectors(self, rel_data: ArchitectAgentRelationship, brain_id):- if not isinstance(rel_data, ArchitectAgentRelationship):- raise TypeError(- f"Expected ArchitectAgentRelationship, got {type(rel_data)}"- )- if hasattr(rel_data, "description") and rel_data.description:- v_rel = self.embeddings.embed_text(rel_data.description)- v_rel.metadata = {- **(self.metadata or {}),- "uuid": rel_data.uuid,- "node_ids": [rel_data.tail.uuid, rel_data.tip.uuid],- "predicate": rel_data.name,- }- if v_rel.embeddings:- v_ids = self.vector_store.add_vectors(- [v_rel], store="relationships", brain_id=brain_id- )- rel_data.properties = {- **(rel_data.properties or {}),- "v_id": v_ids[0],- }- else:- print("[ ! ]Relationship not embedded:", rel_data)- return rel_data.uuid, (- rel_data.properties.get("v_id") if rel_data.properties else None- )--def enrich_kg_from_input(input: str, targeting: Optional[Node] = None, brain_id: str = "default") -> str:@@ -105,6 +32,10 @@ def enrich_kg_from_input(Enrich the knowledge graph from the input."""+ ingestion_manager = IngestionManager(+ embeddings_adapter, vector_store_adapter, graph_adapter+ )+scout_agent = ScoutAgent(llm_small_adapter,cache_adapter,@@ -118,12 +49,11 @@ def enrich_kg_from_input(kg=graph_adapter,vector_store=vector_store_adapter,embeddings=embeddings_adapter,+ ingestion_manager=ingestion_manager,)token_details = []- manager = IngestionManager(embeddings_adapter, vector_store_adapter, graph_adapter)-entities = scout_agent.run(input, targeting=targeting, brain_id=brain_id)token_details.append(token_detail_from_token_counts(@@ -135,7 +65,11 @@ def enrich_kg_from_input()architect_response = architect_agent.run_tooler(- input, entities.entities, targeting=targeting, brain_id=brain_id+ input,+ entities.entities,+ targeting=targeting,+ brain_id=brain_id,+ timeout=360 * len(input),)token_details.append(token_detail_from_token_counts(@@ -146,105 +80,6 @@ def enrich_kg_from_input())- graph_nodes = []-- with ThreadPoolExecutor(max_workers=10) as io_executor:- node_embedding_futures = []-- for entity in entities.entities:- future = io_executor.submit(manager.process_node_vectors, entity, brain_id)- node_embedding_futures.append((future, entity))-- for future, node_data in node_embedding_futures:- try:- future.result(timeout=180)- graph_nodes.append(- Node(- uuid=node_data.uuid,- labels=[node_data.type],- name=node_data.name,- description=node_data.description,- properties=node_data.properties,- )- )- except FutureTimeoutError:- print(- f"[!] Node embedding future timed out for {node_data.name}, skipping"- )- continue- except Exception as e:- print(f"[!] Node embedding future failed for {node_data.name}: {e}")- continue-- graph_adapter.add_nodes(graph_nodes, brain_id=brain_id)-- relationships_list = architect_response- flattened_relationships = []- for item in relationships_list:- if isinstance(item, list):- flattened_relationships.extend(item)- else:- flattened_relationships.append(item)-- rel_embedding_futures: List[Tuple[Future, ArchitectAgentRelationship]] = []- for relationship in flattened_relationships:- if not isinstance(relationship, ArchitectAgentRelationship):- print(f"[!] Skipping invalid relationship type: {type(relationship)}")- continue- future = io_executor.submit(- manager.process_rel_vectors, relationship, brain_id- )- rel_embedding_futures.append((future, relationship))-- for future, relationship in rel_embedding_futures:- try:- v_id, v_rel_id = future.result(timeout=180)- graph_adapter.add_relationship(- Node(- uuid=relationship.tail.uuid,- flow_key=relationship.tail.flow_key,- labels=[relationship.tail.type],- name=relationship.tail.name,- **(- {"happened_at": relationship.tail.happened_at}- if relationship.tail.happened_at- else {}- ),- ),- Predicate(- uuid=relationship.uuid,- flow_key=relationship.flow_key,- name=relationship.name,- description=relationship.description,- properties={- **(relationship.properties or {}),- "v_id": v_rel_id,- },- ),- Node(- uuid=relationship.tip.uuid,- flow_key=relationship.tip.flow_key,- labels=[relationship.tip.type],- name=relationship.tip.name,- **(- {"happened_at": relationship.tip.happened_at}- if relationship.tip.happened_at- else {}- ),- ),- brain_id=brain_id,- )- except FutureTimeoutError:- rel_name = getattr(relationship, "name", "unknown")- print(- f"[!] Relationship embedding future timed out for {rel_name}, skipping"- )- continue- except Exception as e:- rel_name = getattr(relationship, "name", "unknown")- print(f"[!] Relationship embedding future failed for {rel_name}: {e}")- continue-token_details = merge_token_details([scout_agent.token_detail, architect_agent.token_detail])diff --git a/src/core/saving/ingestion_manager.py b/src/core/saving/ingestion_manager.pynew file mode 100644index 0000000..151e50c--- /dev/null+++ b/src/core/saving/ingestion_manager.py@@ -0,0 +1,81 @@+"""+File: /ingestion_manager.py+Project: saving+Created Date: Sunday January 18th 2026+Author: Christian Nonis <[email protected]>+-----+Last Modified: Sunday January 18th 2026 4:04:33 pm+Modified By: Christian Nonis <[email protected]>+-----+"""++from src.adapters.embeddings import EmbeddingsAdapter+from src.adapters.graph import GraphAdapter+from src.adapters.embeddings import VectorStoreAdapter+from src.core.agents.scout_agent import ScoutEntity+from src.core.agents.architect_agent import ArchitectAgentRelationship+++class IngestionManager:+ def __init__(+ self,+ embeddings_adapter: EmbeddingsAdapter,+ vector_store_adapter: VectorStoreAdapter,+ graph_adapter: GraphAdapter,+ ):+ self.embeddings = embeddings_adapter+ self.vector_store = vector_store_adapter+ self.kg = graph_adapter+ self.resolved_cache = {}+ self.metadata = {}++ def process_node_vectors(self, node_data: ScoutEntity, brain_id):+ if node_data.name in self.resolved_cache:+ return node_data.uuid++ v_sub = self.embeddings.embed_text(node_data.name)+ v_sub.metadata = {+ "labels": [node_data.type],+ "name": node_data.name,+ "uuid": node_data.uuid,+ }+ if v_sub.embeddings:+ v_ids = self.vector_store.add_vectors(+ [v_sub], store="nodes", brain_id=brain_id+ )+ node_data.properties = {+ **(node_data.properties or {}),+ "v_id": v_ids[0],+ }+ self.resolved_cache[node_data.name] = v_ids[0]+ return node_data.uuid+ else:+ print("[ ! ]Node not embedded:", node_data)+ return node_data.uuid++ def process_rel_vectors(self, rel_data: ArchitectAgentRelationship, brain_id):+ if not isinstance(rel_data, ArchitectAgentRelationship):+ raise TypeError(+ f"Expected ArchitectAgentRelationship, got {type(rel_data)}"+ )+ if hasattr(rel_data, "description") and rel_data.description:+ v_rel = self.embeddings.embed_text(rel_data.description)+ v_rel.metadata = {+ **(self.metadata or {}),+ "uuid": rel_data.uuid,+ "node_ids": [rel_data.tail.uuid, rel_data.tip.uuid],+ "predicate": rel_data.name,+ }+ if v_rel.embeddings:+ v_ids = self.vector_store.add_vectors(+ [v_rel], store="relationships", brain_id=brain_id+ )+ rel_data.properties = {+ **(rel_data.properties or {}),+ "v_id": v_ids[0],+ }+ else:+ print("[ ! ]Relationship not embedded:", rel_data)+ return rel_data.uuid, (+ rel_data.properties.get("v_id") if rel_data.properties else None+ )diff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.pyindex 4d24988..70c2426 100644--- a/src/lib/neo4j/client.py+++ b/src/lib/neo4j/client.py@@ -3,8 +3,8 @@ File: /client.pyCreated Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>------Last Modified: Saturday December 27th 2025-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Last Modified: Monday January 12th 2026 8:26:26 pm+Modified By: Christian Nonis <[email protected]>-----"""@@ -217,8 +217,11 @@ class Neo4jClient(GraphClient):identification_set_str = f"{{{', '.join(identification_items)}}}"- if node.description is not None:- all_properties["description"] = node.description+ attributes = ["description", "happened_at", "flow_key", "last_updated"]++ for attr in attributes:+ if getattr(node, attr, None):+ all_properties[attr] = getattr(node, attr)property_assignments = []for key, value in all_properties.items():@@ -264,11 +267,31 @@ class Neo4jClient(GraphClient):"""Add a relationship between two nodes to the graph."""++ objects = [subject, to_object, predicate]+ attributes = [+ "properties",+ "description",+ "happened_at",+ "flow_key",+ "last_updated",+ ]++ extra_ops = ""+ for obj in objects:+ for attr in attributes:+ value = getattr(obj, attr, None)+ if value:+ extra_ops += f"""+ SET r.{attr} = {self._format_value(value)}+ """+cypher_query = f"""MATCH (a:{":".join(self._clean_labels(subject.labels))}) WHERE a.name = {self._format_value(subject.name)}MATCH (b:{":".join(self._clean_labels(to_object.labels))}) WHERE b.name = {self._format_value(to_object.name)}MERGE (a)-[r:{":".join(self._clean_labels([predicate.name]))}]->(b)- ON CREATE SET r.description = {self._format_value(predicate.description)}, r.uuid = {self._format_value(predicate.uuid)}+ ON CREATE SET r.description = {self._format_value(predicate.description)}, r.uuid = {self._format_value(predicate.uuid)}, r.v_id = {self._format_value(predicate.properties.get("v_id"))}+ {extra_ops}RETURN a, b"""@@ -1305,5 +1328,27 @@ class Neo4jClient(GraphClient):from_, flattened, vector_store_adapter, brain_id)+ def check_node_existence(+ self,+ uuid: str,+ name: str,+ labels: list[str],+ brain_id: str,+ ) -> bool:+ """+ Check if a node exists in the graph.+ """+ cypher_query = f"""+ MATCH (n:{":".join(self._clean_labels(labels))})+ WHERE n.name = $name+ AND n.uuid = $uuid+ RETURN n+ """+ self.ensure_database(brain_id)+ result = self.driver.execute_query(+ cypher_query, parameters_={"uuid": uuid, "name": name}, database_=brain_id+ )+ return len(result.records) > 0+_neo4j_client = Neo4jClient()diff --git a/src/workers/app.py b/src/workers/app.pyindex 3713860..d05f270 100644--- a/src/workers/app.py+++ b/src/workers/app.py@@ -4,7 +4,7 @@ Created Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>-----Last Modified: Sunday October 19th 2025 9:16:36 am-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Modified By: Christian Nonis <[email protected]>-----"""@@ -45,8 +45,8 @@ ingestion_app.conf.update(broker_connection_timeout=10,result_expires=3600 * 24 * 7,task_track_started=True,- task_time_limit=3600,- task_soft_time_limit=3000,+ task_time_limit=86400,+ task_soft_time_limit=86400,broker_transport_options={"visibility_timeout": 7200,"fanout_prefix": True,diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.pyindex 836990c..db456e7 100644--- a/src/workers/tasks/ingestion.py+++ b/src/workers/tasks/ingestion.py@@ -3,8 +3,8 @@ File: /ingestion.pyCreated Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>------Last Modified: Saturday December 13th 2025-Modified By: the developer formerly known as Christian Nonis at <[email protected]>+Last Modified: Monday January 5th 2026 7:57:27 pm+Modified By: Christian Nonis <[email protected]>-----"""@@ -12,6 +12,12 @@ import jsonimport tomllibfrom pathlib import Pathfrom uuid import uuid4+from typing import List, Tuple+from concurrent.futures import (+ Future,+ ThreadPoolExecutor,+ TimeoutError as FutureTimeoutError,+)from src.constants.data import (KGChangeLogPredicateUpdatedProperty,KGChanges,@@ -22,9 +28,11 @@ from src.constants.data import (TextChunk,KGChangeLogNodePropertiesUpdated,)-from src.constants.kg import Node+from src.constants.kg import Node, Predicate+from src.constants.agents import ArchitectAgentRelationshipfrom src.constants.prompts.misc import NODE_DESCRIPTION_PROMPTfrom src.core.saving.auto_kg import enrich_kg_from_input+from src.core.saving.ingestion_manager import IngestionManagerfrom src.services.api.constants.requests import IngestionStructuredRequestBodyfrom src.services.data.main import data_adapterfrom src.services.input.agents import llm_small_adapter@@ -176,6 +184,193 @@ def ingest_data(self, args: dict):raise+@ingestion_app.task(bind=True)+def process_architect_relationships(self, args: dict):+ """+ Process architect relationships.+ """++ print(f"> Processing {len(args.get('relationships', []))} architect relationships")++ relationships_data: List[dict] = args.get("relationships", [])+ brain_id: str = args.get("brain_id", "default")++ try:+ cache_adapter.set(+ key=f"task:{self.request.id}",+ value=json.dumps(+ {+ "status": "started",+ "task_id": self.request.id,+ "total_relationships": len(relationships_data),+ }+ ),+ brain_id=brain_id,+ expires_in=3600 * 24 * 7,+ )++ ingestion_manager = IngestionManager(+ embeddings_adapter, vector_store_adapter, graph_adapter+ )++ relationships = [+ ArchitectAgentRelationship(**rel_data) for rel_data in relationships_data+ ]++ with ThreadPoolExecutor(max_workers=10) as io_executor:+ rel_embedding_futures: List[Tuple[Future, ArchitectAgentRelationship]] = []+ for relationship in relationships:+ if not isinstance(relationship, ArchitectAgentRelationship):+ print(+ f"[!] Skipping invalid relationship type: {type(relationship)}"+ )+ continue+ future = io_executor.submit(+ ingestion_manager.process_rel_vectors,+ relationship,+ brain_id,+ )+ rel_embedding_futures.append((future, relationship))++ for future, relationship in rel_embedding_futures:+ print(f"> Processing relationship {relationship.name}")+ try:+ v_id, v_rel_id = future.result(timeout=180)++ subject_exists = graph_adapter.check_node_existence(+ uuid=relationship.tail.uuid,+ name=relationship.tail.name,+ labels=[relationship.tail.type],+ brain_id=brain_id,+ )+ object_exists = graph_adapter.check_node_existence(+ uuid=relationship.tip.uuid,+ name=relationship.tip.name,+ labels=[relationship.tip.type],+ brain_id=brain_id,+ )++ node_embedding_futures = []+ print(f"> Subject exists: {subject_exists}")+ print(f"> Object exists: {object_exists}")+ if not subject_exists:+ future = io_executor.submit(+ ingestion_manager.process_node_vectors,+ relationship.tail,+ brain_id,+ )+ node_embedding_futures.append((future, relationship.tail))+ if not object_exists:+ future = io_executor.submit(+ ingestion_manager.process_node_vectors,+ relationship.tip,+ brain_id,+ )+ node_embedding_futures.append((future, relationship.tip))++ graph_nodes = []++ for future, node_data in node_embedding_futures:+ print(f"> Processing node {node_data.name}")+ try:+ future.result(timeout=180)+ graph_nodes.append(+ Node(+ uuid=node_data.uuid,+ labels=[node_data.type],+ name=node_data.name,+ description=node_data.description,+ properties=node_data.properties,+ )+ )+ except FutureTimeoutError:+ print(+ f"[!] Node embedding future timed out for {node_data.name}, skipping"+ )+ continue+ except Exception as e:+ print(+ f"[!] Node embedding future failed for {node_data.name}: {e}"+ )+ continue++ graph_adapter.add_nodes(graph_nodes, brain_id=brain_id)+ print(f"> Added {len(graph_nodes)} nodes")+ graph_adapter.add_relationship(+ Node(+ uuid=relationship.tail.uuid,+ flow_key=relationship.tail.flow_key,+ labels=[relationship.tail.type],+ name=relationship.tail.name,+ **(+ {"happened_at": relationship.tail.happened_at}+ if relationship.tail.happened_at+ else {}+ ),+ ),+ Predicate(+ uuid=relationship.uuid,+ flow_key=relationship.flow_key,+ name=relationship.name,+ description=relationship.description,+ properties={+ **(relationship.properties or {}),+ "v_id": v_rel_id,+ },+ ),+ Node(+ uuid=relationship.tip.uuid,+ flow_key=relationship.tip.flow_key,+ labels=[relationship.tip.type],+ name=relationship.tip.name,+ **(+ {"happened_at": relationship.tip.happened_at}+ if relationship.tip.happened_at+ else {}+ ),+ ),+ brain_id=brain_id,+ )+ except FutureTimeoutError:+ rel_name = getattr(relationship, "name", "unknown")+ print(+ f"[!] Relationship embedding future timed out for {rel_name}, skipping"+ )+ continue+ except Exception as e:+ rel_name = getattr(relationship, "name", "unknown")+ print(+ f"[!] Relationship embedding future failed for {rel_name}: {e}"+ )+ continue++ cache_adapter.set(+ key=f"task:{self.request.id}",+ value=json.dumps({"status": "completed", "task_id": self.request.id}),+ brain_id=brain_id,+ expires_in=3600 * 24 * 7,+ )++ return self.request.id++ except Exception as e:+ error_result = {+ "status": "failed",+ "task_id": self.request.id,+ "error": str(e),+ "args": args,+ }++ cache_adapter.set(+ key=f"task:{self.request.id}",+ value=json.dumps(error_result),+ brain_id=brain_id,+ expires_in=3600 * 24 * 7,+ )++ raise++@ingestion_app.task(bind=True)def ingest_structured_data(self, args: dict):"""