BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
added improvements
marcorasera54a9f8e1f86054
diff --git a/src/services/api/routes/retrieve.py b/src/services/api/routes/retrieve.pyindex 7a02f93..ec2040e 100644--- a/src/services/api/routes/retrieve.py+++ b/src/services/api/routes/retrieve.py@@ -159,20 +159,23 @@ async def get_entities()@retrieve_router.get(path="/structured-data/types")-async def get_structured_data_types():+async def get_structured_data_types(+ brain_id: str = "default",+):"""Get all unique types from structured data."""- return await get_structured_data_types_controller()+ return await get_structured_data_types_controller(brain_id)@retrieve_router.get(path="/structured-data/{id}")async def get_structured_data_by_id(id: str,+ brain_id: str = "default",):"""Get structured data by ID."""- return await get_structured_data_by_id_controller(id)+ return await get_structured_data_by_id_controller(id, brain_id)@retrieve_router.get(path="/structured-data")async def get_structured_data_list(@@ -180,30 +183,33 @@ async def get_structured_data_list(skip: int = 0,types: Optional[str] = None,query_text: Optional[str] = None,+ brain_id: str = "default",):"""Get a list of structured data."""if types:types = types.split(",")- return await get_structured_data_list_controller(limit, skip, types, query_text)+ return await get_structured_data_list_controller(limit, skip, types, query_text, brain_id)@retrieve_router.get(path="/observations/labels")-async def get_observation_labels():+async def get_observation_labels(+ brain_id: str = "default",+):"""Get all unique labels from observations."""- return await get_observation_labels_controller()+ return await get_observation_labels_controller(brain_id)@retrieve_router.get(path="/observations/{id}")async def get_observation_by_id(id: str,+ brain_id: str = "default",):"""Get observation by ID."""- return await get_observation_by_id_controller(id)-+ return await get_observation_by_id_controller(id, brain_id)@retrieve_router.get(path="/observations")async def get_observations_list(@@ -212,10 +218,11 @@ async def get_observations_list(resource_id: Optional[str] = None,labels: Optional[str] = None,query_text: Optional[str] = None,+ brain_id: str = "default",):"""Get a list of observations."""if labels:labels = labels.split(",")- return await get_observations_list_controller(limit, skip, resource_id, labels, query_text)\ No newline at end of file+ return await get_observations_list_controller(limit, skip, resource_id, labels, query_text, brain_id)\ No newline at end of filediff --git a/src/services/api/routes/tasks.py b/src/services/api/routes/tasks.pyindex f8edbac..5c44e17 100644--- a/src/services/api/routes/tasks.py+++ b/src/services/api/routes/tasks.py@@ -9,12 +9,11 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@"""import jsonfrom src.utils.logging import log-from fastapi import APIRouter, HTTPException+from fastapi import APIRouterfrom src.services.kg_agent.main import cache_adaptertasks_router = APIRouter(prefix="/tasks", tags=["tasks"])-@tasks_router.get("/{task_id}")async def get_task(task_id: str, brain_id: str = "default"):"""@@ -22,18 +21,19 @@ async def get_task(task_id: str, brain_id: str = "default"):"""try:str_result = cache_adapter.get(f"task:{task_id}", brain_id=brain_id)-if str_result is None:- raise HTTPException(status_code=404, detail="Task not found")-+ return {+ "task_id": task_id,+ "status": "pending",+ "result": {"message": "Task is still processing or not found"}+ }if isinstance(str_result, bytes):result = json.loads(str_result.decode("utf-8"))else:result = json.loads(str_result)-return {"task_id": task_id,- "status": result.get("status", "pending"),+ "status": result.get("status", "unknown"),"result": result}except Exception as e:@@ -41,5 +41,8 @@ async def get_task(task_id: str, brain_id: str = "default"):return {"task_id": task_id,"status": "error",- "result": {"error": str(e)}+ "result": {+ "error": str(e),+ "error_type": type(e).__name__+ }}\ No newline at end of filediff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.pyindex a07cd34..9f5cb4e 100644--- a/src/workers/tasks/ingestion.py+++ b/src/workers/tasks/ingestion.py@@ -43,108 +43,125 @@ def ingest_data(self, args: dict):"""Ingest data into the database."""- payload = IngestionTaskArgs(**args)+ try:+ payload = IngestionTaskArgs(**args)- payload.meta_keys = (- {- f"{k.replace(' ', '_').lower()}": v- for k, v in payload.meta_keys.items()- if v is not None- }- if payload.meta_keys- else None- )+ payload.meta_keys = (+ {+ f"{k.replace(' ', '_').lower()}": v+ for k, v in payload.meta_keys.items()+ if v is not None+ }+ if payload.meta_keys+ else None+ )++ payload.identification_params = (+ {+ f"{k.replace(' ', '_').lower()}": v+ for k, v in payload.identification_params.items()+ if v is not None+ }+ if payload.identification_params+ else None+ )- payload.identification_params = (- {- f"{k.replace(' ', '_').lower()}": v- for k, v in payload.identification_params.items()- if v is not None+ # ================================================+ # --------------- Data Saving --------------------+ # ================================================+ text_chunk = data_adapter.save_text_chunk(+ TextChunk(+ text=(+ payload.data.text_data+ if payload.data.data_type == IngestionTaskDataType.TEXT.value+ else json.dumps(payload.data.json_data)+ ),+ metadata=payload.meta_keys,+ ),+ brain_id=payload.brain_id,+ )+ text_chunk_vector = embeddings_adapter.embed_text(text_chunk.text)+ text_chunk_vector.metadata = {+ **payload.meta_keys,+ "resource_id": text_chunk.id,}- if payload.identification_params- else None- )+ vector_store_adapter.add_vectors(+ [text_chunk_vector],+ "data",+ )- # ================================================- # --------------- Data Saving --------------------- # ================================================- text_chunk = data_adapter.save_text_chunk(- TextChunk(+ # ================================================+ # --------------- Observations -------------------+ # ================================================+ observations = observations_agent.observe(text=(payload.data.text_dataif payload.data.data_type == IngestionTaskDataType.TEXT.valueelse json.dumps(payload.data.json_data)),- metadata=payload.meta_keys,- ),- brain_id=payload.brain_id,- )- text_chunk_vector = embeddings_adapter.embed_text(text_chunk.text)- text_chunk_vector.metadata = {- **payload.meta_keys,- "resource_id": text_chunk.id,- }- vector_store_adapter.add_vectors(- [text_chunk_vector],- "data",- )+ observate_for=payload.observate_for,+ )- # ================================================- # --------------- Observations -------------------- # ================================================- observations = observations_agent.observe(- text=(- payload.data.text_data- if payload.data.data_type == IngestionTaskDataType.TEXT.value- else json.dumps(payload.data.json_data)- ),- observate_for=payload.observate_for,- )+ data_adapter.save_observations(+ [+ Observation(+ text=observation,+ metadata=payload.meta_keys,+ resource_id=text_chunk.id,+ )+ for observation in observations+ ]+ )- data_adapter.save_observations(- [- Observation(- text=observation,- metadata=payload.meta_keys,- resource_id=text_chunk.id,- )- for observation in observations- ]- )+ # ================================================+ # ------------ Triplet Extraction ----------------+ # ================================================+ information = f"""+ {payload.data.text_data+ if payload.data.data_type == IngestionTaskDataType.TEXT.value+ else json.dumps(payload.data.json_data)}++ Annotations:+ {observations}+ """- # ================================================- # ------------ Triplet Extraction ----------------- # ================================================- information = f"""- {payload.data.text_data- if payload.data.data_type == IngestionTaskDataType.TEXT.value- else json.dumps(payload.data.json_data)}-- Annotations:- {observations}- """+ kg_agent.update_kg(+ information=information,+ metadata={**payload.meta_keys, "resource_id": text_chunk.id},+ identification_params=payload.identification_params,+ preferred_entities=payload.preferred_extraction_entities,+ brain_id=payload.brain_id,+ )- kg_agent.update_kg(- information=information,- metadata={**payload.meta_keys, "resource_id": text_chunk.id},- identification_params=payload.identification_params,- preferred_entities=payload.preferred_extraction_entities,- brain_id=payload.brain_id,- )+ result = {+ "status": "completed",+ "task_id": self.request.id,+ "text_chunk_id": text_chunk.id,+ "observations_count": len(observations)+ }++ cache_adapter.set(+ f"task:{self.request.id}",+ json.dumps(result),+ expires_in=3600+ )- result = {- "status": "completed",- "task_id": self.request.id,- "text_chunk_id": text_chunk.id,- "observations_count": len(observations)- }+ return self.request.id- cache_adapter.set(- f"task:{self.request.id}",- json.dumps(result)- )-- return self.request.id+ except Exception as e:+ error_result = {+ "status": "failed",+ "task_id": self.request.id,+ "error": str(e),+ }++ cache_adapter.set(+ f"task:{self.request.id}",+ json.dumps(error_result),+ expires_in=3600+ )++ raise@ingestion_app.task(bind=True)@@ -152,154 +169,171 @@ def ingest_structured_data(self, args: dict):"""Ingest structured data into the database."""- payload = IngestionStructuredRequestBody(**args)+ try:+ payload = IngestionStructuredRequestBody(**args)- for element in payload.data:- uuid = str(uuid4())- node = graph_adapter.add_nodes(- [- Node(- uuid=uuid,- labels=element.types,- name=element.identification_params.name,- properties={- **(element.json_data if element.json_data else {}),- **(- element.identification_params.model_dump(mode="json")- if element.identification_params- else {}- ),- },- )- ],- brain_id=payload.brain_id,- identification_params=element.identification_params.model_dump(mode="json"),- metadata=element.textual_data,- )[0]-- # Saving the node properties updated change- kg_changes = KGChanges(- type=KGChangesType.NODE_PROPERTIES_UPDATED,- change=KGChangeLogNodePropertiesUpdated(- node=PartialNode(- **node.model_dump(mode="json"),- ),- properties=[- KGChangeLogPredicateUpdatedProperty(- property=property,- previous_value=None,- new_value=node.properties[property],+ for element in payload.data:+ uuid = str(uuid4())+ node = graph_adapter.add_nodes(+ [+ Node(+ uuid=uuid,+ labels=element.types,+ name=element.identification_params.name,+ properties={+ **(element.json_data if element.json_data else {}),+ **(+ element.identification_params.model_dump(mode="json")+ if element.identification_params+ else {}+ ),+ },)- for property in node.properties],- ),- )- data_adapter.save_kg_changes(kg_changes, brain_id=payload.brain_id)-- vector = embeddings_adapter.embed_text(node.name)- vector.id = node.uuid- vector.metadata = {- "labels": node.labels,- "name": node.name,- "uuid": node.uuid,- }- vector_store_adapter.add_vectors(vectors=[vector], store="nodes")+ brain_id=payload.brain_id,+ identification_params=element.identification_params.model_dump(mode="json"),+ metadata=element.textual_data,+ )[0]- data_adapter.save_structured_data(- StructuredData(- id=uuid,- data=element.json_data,- types=element.types,- metadata={- "identification_params": element.identification_params,- "textual_data": element.textual_data,- "labels": element.types,- "name": element.identification_params.name,- },- ),- brain_id=payload.brain_id,- )- vector = embeddings_adapter.embed_text(- "\n".join(- [- (- v- if isinstance(v, str)- else (", ".join(v) if isinstance(v, list) else str(v))- )- for v in element.textual_data.values()- ]+ # Saving the node properties updated change+ kg_changes = KGChanges(+ type=KGChangesType.NODE_PROPERTIES_UPDATED,+ change=KGChangeLogNodePropertiesUpdated(+ node=PartialNode(+ **node.model_dump(mode="json"),+ ),+ properties=[+ KGChangeLogPredicateUpdatedProperty(+ property=property,+ previous_value=None,+ new_value=node.properties[property],+ )+ for property in node.properties+ ],+ ),)- + "\n"- + (", ".join(node.labels) if isinstance(node.labels, list) else node.labels)- + "\n"- + node.name,- )- vector.id = node.uuid- vector.metadata = {- "labels": node.labels,- "name": node.name,- "uuid": node.uuid,- }- vector_store_adapter.add_vectors(vectors=[vector], store="data")+ data_adapter.save_kg_changes(kg_changes, brain_id=payload.brain_id)- text_content = "\n".join(- [- (- f"{k}: {v}"- if isinstance(v, str)- else (f"{k}: {', '.join(v) if isinstance(v, list) else str(v)}")+ vector = embeddings_adapter.embed_text(node.name)+ vector.id = node.uuid+ vector.metadata = {+ "labels": node.labels,+ "name": node.name,+ "uuid": node.uuid,+ }+ vector_store_adapter.add_vectors(vectors=[vector], store="nodes")++ data_adapter.save_structured_data(+ StructuredData(+ id=uuid,+ data=element.json_data,+ types=element.types,+ metadata={+ "identification_params": element.identification_params,+ "textual_data": element.textual_data,+ "labels": element.types,+ "name": element.identification_params.name,+ },+ ),+ brain_id=payload.brain_id,+ )+ vector = embeddings_adapter.embed_text(+ "\n".join(+ [+ (+ v+ if isinstance(v, str)+ else (", ".join(v) if isinstance(v, list) else str(v))+ )+ for v in element.textual_data.values()+ ])- for k, v in element.textual_data.items()- ]- )- observations = observations_agent.observe(- text=text_content,- observate_for=payload.observate_for,- )- for obs in observations:- obs_vector = embeddings_adapter.embed_text(obs)- obs_vector.id = str(uuid4())- obs_vector.metadata = {- "resource_id": node.uuid,+ + "\n"+ + (", ".join(node.labels) if isinstance(node.labels, list) else node.labels)+ + "\n"+ + node.name,+ )+ vector.id = node.uuid+ vector.metadata = {"labels": node.labels,"name": node.name,+ "uuid": node.uuid,}- vector_store_adapter.add_vectors(vectors=[obs_vector], store="observations")- if len(observations) > 0:- data_adapter.save_observations(+ vector_store_adapter.add_vectors(vectors=[vector], store="data")++ text_content = "\n".join([- Observation(- id=node.uuid,- text=obs,- metadata={- "labels": node.labels,- "name": node.name,- },- resource_id=node.uuid,+ (+ f"{k}: {v}"+ if isinstance(v, str)+ else (f"{k}: {', '.join(v) if isinstance(v, list) else str(v)}"))- for obs in observations- ],+ for k, v in element.textual_data.items()+ ]+ )+ observations = observations_agent.observe(+ text=text_content,+ observate_for=payload.observate_for,+ )+ for obs in observations:+ obs_vector = embeddings_adapter.embed_text(obs)+ obs_vector.id = str(uuid4())+ obs_vector.metadata = {+ "resource_id": node.uuid,+ "labels": node.labels,+ "name": node.name,+ }+ vector_store_adapter.add_vectors(vectors=[obs_vector], store="observations")+ if len(observations) > 0:+ data_adapter.save_observations(+ [+ Observation(+ id=node.uuid,+ text=obs,+ metadata={+ "labels": node.labels,+ "name": node.name,+ },+ resource_id=node.uuid,+ )+ for obs in observations+ ],+ brain_id=payload.brain_id,+ )++ # TODO: + vectorize the predicates and new nodes + save changelog&vectors+ kg_agent.structured_update_kg(+ main_node=node,+ textual_data=element.textual_data,+ identification_params=element.identification_params.model_dump(mode="json"),brain_id=payload.brain_id,)- # TODO: + vectorize the predicates and new nodes + save changelog&vectors- kg_agent.structured_update_kg(- main_node=node,- textual_data=element.textual_data,- identification_params=element.identification_params.model_dump(mode="json"),- brain_id=payload.brain_id,+ result = {+ "status": "completed",+ "task_id": self.request.id,+ "ingested_count": len(payload.data)+ }++ cache_adapter.set(+ f"task:{self.request.id}",+ json.dumps(result),+ expires_in=3600)- result = {- "status": "completed",- "task_id": self.request.id,- "ingested_count": len(payload.data)- }+ return self.request.id- cache_adapter.set(- f"task:{self.request.id}",- json.dumps(result)- )-- return self.request.id+ except Exception as e:+ error_result = {+ "status": "failed",+ "task_id": self.request.id,+ "error": str(e),+ }++ cache_adapter.set(+ f"task:{self.request.id}",+ json.dumps(error_result),+ expires_in=3600+ )++ raise\ No newline at end of file