BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
implemented tasks, observations, structured_data logic
marcorasera54c2382fe08b7e
diff --git a/.gitignore b/.gitignoreindex 1899141..2a0da12 100644--- a/.gitignore+++ b/.gitignore@@ -6,4 +6,5 @@ __pycache__distcommon.egg-info.venv-.env.development\ No newline at end of file+.env.development+venv/\ No newline at end of filediff --git a/src/adapters/data.py b/src/adapters/data.pyindex 9fd406f..c09edb1 100644--- a/src/adapters/data.py+++ b/src/adapters/data.py@@ -3,7 +3,7 @@ File: /data.pyCreated Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>------Last Modified: Sunday October 19th 2025 9:00:42 am+Last Modified: Saturday December 13th 2025Modified By: the developer formerly known as Christian Nonis at <[email protected]>-----"""@@ -90,3 +90,53 @@ class DataAdapter:Save a KG changes to the data client."""return self.data.save_kg_changes(kg_changes, brain_id)++ def get_structured_data_by_id(+ self, id: str, brain_id: str = "default"+ ) -> StructuredData:+ """+ Get structured data by ID.+ """+ return self.data.get_structured_data_by_id(id, brain_id)++ def get_structured_data_list(+ self, brain_id: str = "default", limit: int = 10, skip: int = 0, types: list[str] = None, query_text: str = None+ ) -> list[StructuredData]:+ """+ Get a list of structured data.+ """+ return self.data.get_structured_data_list(brain_id, limit, skip, types, query_text)++ def get_structured_data_types(self, brain_id: str = "default") -> list[str]:+ """+ Get all unique types from structured data.+ """+ return self.data.get_structured_data_types(brain_id)++ def get_observation_by_id(+ self, id: str, brain_id: str = "default"+ ) -> Observation:+ """+ Get observation by ID.+ """+ return self.data.get_observation_by_id(id, brain_id)++ def get_observations_list(+ self,+ brain_id: str = "default",+ limit: int = 10,+ skip: int = 0,+ resource_id: str = None,+ labels: list[str] = None,+ query_text: str = None+ ) -> list[Observation]:+ """+ Get a list of observations.+ """+ return self.data.get_observations_list(brain_id, limit, skip, resource_id, labels, query_text)++ def get_observation_labels(self, brain_id: str = "default") -> list[str]:+ """+ Get all unique labels from observations.+ """+ return self.data.get_observation_labels(brain_id)\ No newline at end of filediff --git a/src/adapters/interfaces/data.py b/src/adapters/interfaces/data.pyindex 24717db..fd741eb 100644--- a/src/adapters/interfaces/data.py+++ b/src/adapters/interfaces/data.py@@ -3,7 +3,7 @@ File: /data.pyCreated Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>------Last Modified: Sunday October 19th 2025 9:01:13 am+Last Modified: Saturday December 13th 2025Modified By: the developer formerly known as Christian Nonis at <[email protected]>-----"""@@ -98,3 +98,55 @@ class DataClient(ABC):Save a KG changes to the data client."""raise NotImplementedError("save_kg_changes method not implemented")++ @abstractmethod+ def get_structured_data_by_id(self, id: str, brain_id: str) -> StructuredData:+ """+ Get structured data by ID.+ """+ raise NotImplementedError("get_structured_data_by_id method not implemented")++ @abstractmethod+ def get_structured_data_list(+ self, brain_id: str, limit: int = 10, skip: int = 0, types: list[str] = None, query_text: str = None+ ) -> list[StructuredData]:+ """+ Get a list of structured data.+ """+ raise NotImplementedError("get_structured_data_list method not implemented")++ @abstractmethod+ def get_structured_data_types(self, brain_id: str) -> list[str]:+ """+ Get all unique types from structured data.+ """+ raise NotImplementedError("get_structured_data_types method not implemented")++ @abstractmethod+ def get_observation_by_id(self, id: str, brain_id: str) -> Observation:+ """+ Get observation by ID.+ """+ raise NotImplementedError("get_observation_by_id method not implemented")++ @abstractmethod+ def get_observations_list(+ self,+ brain_id: str,+ limit: int = 10,+ skip: int = 0,+ resource_id: str = None,+ labels: list[str] = None,+ query_text: str = None+ ) -> list[Observation]:+ """+ Get a list of observations.+ """+ raise NotImplementedError("get_observations_list method not implemented")++ @abstractmethod+ def get_observation_labels(self, brain_id: str) -> list[str]:+ """+ Get all unique labels from observations.+ """+ raise NotImplementedError("get_observation_labels method not implemented")\ No newline at end of filediff --git a/src/lib/mongo/client.py b/src/lib/mongo/client.pyindex 124e155..a7a985d 100644--- a/src/lib/mongo/client.py+++ b/src/lib/mongo/client.py@@ -3,7 +3,7 @@ File: /client.pyCreated Date: Saturday October 25th 2025Author: Christian Nonis <[email protected]>------Last Modified: Saturday October 25th 2025 11:55:47 am+Last Modified: Saturday December 13th 2025Modified By: the developer formerly known as Christian Nonis at <[email protected]>-----"""@@ -168,5 +168,117 @@ class MongoClient(DataClient):collection.insert_one(kg_changes.model_dump(mode="json"))return kg_changes+ def get_structured_data_by_id(+ self, id: str, brain_id: str+ ) -> StructuredData:+ collection = self.get_collection("structured_data", database=brain_id)+ result = collection.find_one({"id": id})+ if not result:+ return None+ return StructuredData(+ id=result["id"],+ data=result["data"],+ types=result["types"],+ metadata=result.get("metadata", None),+ inserted_at=result.get("inserted_at", None),+ )++ def get_structured_data_list(+ self, brain_id: str, limit: int = 10, skip: int = 0, types: list[str] = None, query_text: str = None+ ) -> list[StructuredData]:+ collection = self.get_collection("structured_data", database=brain_id)+ query = {}+ if types:+ query["types"] = {"$in": types}++ if query_text:+ query["$or"] = [+ {"data": {"$regex": query_text, "$options": "i"}},+ {"types": {"$regex": query_text, "$options": "i"}},+ {"metadata": {"$regex": query_text, "$options": "i"}}+ ]++ results = collection.find(query).skip(skip).limit(limit)+ return [+ StructuredData(+ id=result["id"],+ data=result["data"],+ types=result["types"],+ metadata=result.get("metadata", None),+ inserted_at=result.get("inserted_at", None),+ )+ for result in results+ ]++ def get_structured_data_types(self, brain_id: str) -> list[str]:+ collection = self.get_collection("structured_data", database=brain_id)+ pipeline = [+ {"$unwind": "$types"},+ {"$group": {"_id": "$types"}},+ {"$sort": {"_id": 1}}+ ]+ results = collection.aggregate(pipeline)+ return [result["_id"] for result in results]++ def get_observation_by_id(+ self, id: str, brain_id: str+ ) -> Observation:+ collection = self.get_collection("observations", database=brain_id)+ result = collection.find_one({"id": id})+ if not result:+ return None+ return Observation(+ id=result["id"],+ text=result["text"],+ metadata=result.get("metadata", None),+ resource_id=result["resource_id"],+ inserted_at=result.get("inserted_at", None),+ )++ def get_observations_list(+ self,+ brain_id: str,+ limit: int = 10,+ skip: int = 0,+ resource_id: str = None,+ labels: list[str] = None,+ query_text: str = None+ ) -> list[Observation]:+ collection = self.get_collection("observations", database=brain_id)+ query = {}+ if resource_id:+ query["resource_id"] = resource_id+ if labels:+ query["metadata.labels"] = {"$in": labels}++ if query_text:+ query["$or"] = [+ {"text": {"$regex": query_text, "$options": "i"}},+ {"resource_id": {"$regex": query_text, "$options": "i"}},+ {"metadata": {"$regex": query_text, "$options": "i"}}+ ]++ results = collection.find(query).skip(skip).limit(limit)+ return [+ Observation(+ id=result["id"],+ text=result["text"],+ metadata=result.get("metadata", None),+ resource_id=result["resource_id"],+ inserted_at=result.get("inserted_at", None),+ )+ for result in results+ ]++ def get_observation_labels(self, brain_id: str) -> list[str]:+ collection = self.get_collection("observations", database=brain_id)+ pipeline = [+ {"$match": {"metadata.labels": {"$exists": True}}},+ {"$unwind": "$metadata.labels"},+ {"$group": {"_id": "$metadata.labels"}},+ {"$sort": {"_id": 1}}+ ]+ results = collection.aggregate(pipeline)+ return [result["_id"] for result in results]_mongo_client = MongoClient()diff --git a/src/services/api/app.py b/src/services/api/app.pyindex 4679af1..7cf6e9a 100644--- a/src/services/api/app.py+++ b/src/services/api/app.py@@ -20,6 +20,7 @@ from src.services.api.routes.ingest import ingest_routerfrom src.services.api.routes.retrieve import retrieve_routerfrom src.services.api.routes.meta import meta_routerfrom src.services.api.routes.system import system_router+from src.services.api.routes.tasks import tasks_routerapp = FastAPI(debug=os.getenv("ENV") == "development")@@ -38,7 +39,7 @@ app.include_router(ingest_router)app.include_router(retrieve_router)app.include_router(meta_router)app.include_router(system_router)-+app.include_router(tasks_router)@app.get("/")async def root():diff --git a/src/services/api/controllers/observations.py b/src/services/api/controllers/observations.pynew file mode 100644index 0000000..77115c6--- /dev/null+++ b/src/services/api/controllers/observations.py@@ -0,0 +1,83 @@+"""+File: /observations.py+Created Date: Saturday December 6th 2025+Author: Christian Nonis <[email protected]>+-----+Last Modified: Saturday December 13th 2025+Modified By: the developer formerly known as Christian Nonis at <[email protected]>+-----+"""++import asyncio+from typing import Optional+from fastapi import HTTPException+from starlette.responses import JSONResponse+from src.services.data.main import data_adapter+++async def get_observation_by_id(+ id: str, brain_id: str = "default"+):+ """+ Get observation by ID.+ """+ def _get_observation():+ observation = data_adapter.get_observation_by_id(id, brain_id)+ if not observation:+ raise HTTPException(status_code=404, detail="Observation not found")+ return observation++ result = await asyncio.to_thread(_get_observation)++ return JSONResponse(+ content={+ "message": "Observation retrieved successfully",+ "observation": result.model_dump(mode="json")+ }+ )+++async def get_observations_list(+ limit: int = 10,+ skip: int = 0,+ resource_id: Optional[str] = None,+ labels: Optional[list[str]] = None,+ query_text: Optional[str] = None,+ brain_id: str = "default",+):+ """+ Get a list of observations.+ """+ def _get_list():+ return data_adapter.get_observations_list(+ brain_id, limit, skip, resource_id, labels, query_text+ )++ results = await asyncio.to_thread(_get_list)++ return JSONResponse(+ content={+ "message": "Observations list retrieved successfully",+ "observations": [r.model_dump(mode="json") for r in results],+ "count": len(results)+ }+ )++async def get_observation_labels(+ brain_id: str = "default",+):+ """+ Get all unique labels from observations.+ """+ def _get_labels():+ return data_adapter.get_observation_labels(brain_id)++ results = await asyncio.to_thread(_get_labels)++ return JSONResponse(+ content={+ "message": "Observation labels retrieved successfully",+ "labels": results,+ "count": len(results)+ }+ )\ No newline at end of filediff --git a/src/services/api/controllers/structured_data.py b/src/services/api/controllers/structured_data.pynew file mode 100644index 0000000..5c65913--- /dev/null+++ b/src/services/api/controllers/structured_data.py@@ -0,0 +1,79 @@+"""+File: /structured_data.py+Created Date: Saturday December 6th 2025+Author: Christian Nonis <[email protected]>+-----+Last Modified: Saturday December 13th 2025+Modified By: the developer formerly known as Christian Nonis at <[email protected]>+-----+"""++import asyncio+from typing import Optional+from fastapi import HTTPException+from starlette.responses import JSONResponse+from src.services.data.main import data_adapter++async def get_structured_data_by_id(+ id: str, brain_id: str = "default"+):+ """+ Get structured data by ID.+ """+ def _get_data():+ data = data_adapter.get_structured_data_by_id(id, brain_id)+ if not data:+ raise HTTPException(status_code=404, detail="Structured data not found")+ return data++ result = await asyncio.to_thread(_get_data)++ return JSONResponse(+ content={+ "message": "Structured data retrieved successfully",+ "data": result.model_dump(mode="json")+ }+ )+++async def get_structured_data_list(+ limit: int = 10,+ skip: int = 0,+ types: Optional[list[str]] = None,+ query_text: Optional[str] = None,+ brain_id: str = "default",+):+ """+ Get a list of structured data.+ """+ def _get_list():+ return data_adapter.get_structured_data_list(brain_id, limit, skip, types, query_text)++ results = await asyncio.to_thread(_get_list)++ return JSONResponse(+ content={+ "message": "Structured data list retrieved successfully",+ "data": [r.model_dump(mode="json") for r in results],+ "count": len(results)+ }+ )++async def get_structured_data_types(+ brain_id: str = "default",+):+ """+ Get all unique types from structured data.+ """+ def _get_types():+ return data_adapter.get_structured_data_types(brain_id)++ results = await asyncio.to_thread(_get_types)++ return JSONResponse(+ content={+ "message": "Structured data types retrieved successfully",+ "types": results,+ "count": len(results)+ }+ )\ No newline at end of filediff --git a/src/services/api/routes/retrieve.py b/src/services/api/routes/retrieve.pyindex bb3112d..7a02f93 100644--- a/src/services/api/routes/retrieve.py+++ b/src/services/api/routes/retrieve.py@@ -3,7 +3,7 @@ File: /retrieve.pyCreated Date: Saturday October 25th 2025Author: Christian Nonis <[email protected]>------Last Modified: Saturday October 25th 2025 12:32:21 pm+Last Modified: Saturday December 13th 2025Modified By: the developer formerly known as Christian Nonis at <[email protected]>-----"""@@ -26,6 +26,16 @@ from src.services.api.controllers.retrieve import (from src.services.api.controllers.retrieve import (retrieve_data as retrieve_data_controller,)+from src.services.api.controllers.structured_data import (+ get_structured_data_by_id as get_structured_data_by_id_controller,+ get_structured_data_list as get_structured_data_list_controller,+ get_structured_data_types as get_structured_data_types_controller,+)+from src.services.api.controllers.observations import (+ get_observation_by_id as get_observation_by_id_controller,+ get_observations_list as get_observations_list_controller,+ get_observation_labels as get_observation_labels_controller,+)retrieve_router = APIRouter(prefix="/retrieve", tags=["retrieve"])@@ -147,3 +157,65 @@ async def get_entities(return await retrieve_get_entities_controller(limit, skip, node_labels, query_text, brain_id)++@retrieve_router.get(path="/structured-data/types")+async def get_structured_data_types():+ """+ Get all unique types from structured data.+ """+ return await get_structured_data_types_controller()++@retrieve_router.get(path="/structured-data/{id}")+async def get_structured_data_by_id(+ id: str,+):+ """+ Get structured data by ID.+ """+ return await get_structured_data_by_id_controller(id)++@retrieve_router.get(path="/structured-data")+async def get_structured_data_list(+ limit: int = 10,+ skip: int = 0,+ types: Optional[str] = None,+ query_text: Optional[str] = None,+):+ """+ Get a list of structured data.+ """+ if types:+ types = types.split(",")+ return await get_structured_data_list_controller(limit, skip, types, query_text)++@retrieve_router.get(path="/observations/labels")+async def get_observation_labels():+ """+ Get all unique labels from observations.+ """+ return await get_observation_labels_controller()++@retrieve_router.get(path="/observations/{id}")+async def get_observation_by_id(+ id: str,+):+ """+ Get observation by ID.+ """+ return await get_observation_by_id_controller(id)+++@retrieve_router.get(path="/observations")+async def get_observations_list(+ limit: int = 10,+ skip: int = 0,+ resource_id: Optional[str] = None,+ labels: Optional[str] = None,+ query_text: Optional[str] = None,+):+ """+ Get a list of observations.+ """+ if labels:+ labels = labels.split(",")+ return await get_observations_list_controller(limit, skip, resource_id, labels, query_text)\ No newline at end of filediff --git a/src/services/api/routes/tasks.py b/src/services/api/routes/tasks.pynew file mode 100644index 0000000..f8edbac--- /dev/null+++ b/src/services/api/routes/tasks.py@@ -0,0 +1,45 @@+"""+File: /tasks.py+Created Date: Saturday December 13th 2025+Author: Christian Nonis <[email protected]>+-----+Last Modified: Saturday December 13th 2025+Modified By: the developer formerly known as Christian Nonis at <[email protected]>+-----+"""+import json+from src.utils.logging import log+from fastapi import APIRouter, HTTPException+from src.services.kg_agent.main import cache_adapter++tasks_router = APIRouter(prefix="/tasks", tags=["tasks"])+++@tasks_router.get("/{task_id}")+async def get_task(task_id: str, brain_id: str = "default"):+ """+ Get the result of a task by its ID.+ """+ try:+ str_result = cache_adapter.get(f"task:{task_id}", brain_id=brain_id)++ if str_result is None:+ raise HTTPException(status_code=404, detail="Task not found")++ if isinstance(str_result, bytes):+ result = json.loads(str_result.decode("utf-8"))+ else:+ result = json.loads(str_result)++ return {+ "task_id": task_id,+ "status": result.get("status", "pending"),+ "result": result+ }+ except Exception as e:+ log(f"Error in get_task: {type(e).__name__}: {str(e)}")+ return {+ "task_id": task_id,+ "status": "error",+ "result": {"error": str(e)}+ }\ No newline at end of filediff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.pyindex aad08a1..a07cd34 100644--- a/src/workers/tasks/ingestion.py+++ b/src/workers/tasks/ingestion.py@@ -3,7 +3,7 @@ File: /ingestion.pyCreated Date: Sunday October 19th 2025Author: Christian Nonis <[email protected]>------Last Modified: Sunday October 19th 2025 12:14:21 pm+Last Modified: Saturday December 13th 2025Modified By: the developer formerly known as Christian Nonis at <[email protected]>-----"""@@ -35,7 +35,8 @@ from src.constants.tasks.ingestion import (IngestionTaskArgs,IngestionTaskDataType,)-+import json+from src.services.kg_agent.main import cache_adapter@ingestion_app.task(bind=True)def ingest_data(self, args: dict):@@ -131,6 +132,18 @@ def ingest_data(self, args: dict):brain_id=payload.brain_id,)+ result = {+ "status": "completed",+ "task_id": self.request.id,+ "text_chunk_id": text_chunk.id,+ "observations_count": len(observations)+ }++ cache_adapter.set(+ f"task:{self.request.id}",+ json.dumps(result)+ )+return self.request.id@@ -278,4 +291,15 @@ def ingest_structured_data(self, args: dict):brain_id=payload.brain_id,)+ result = {+ "status": "completed",+ "task_id": self.request.id,+ "ingested_count": len(payload.data)+ }++ cache_adapter.set(+ f"task:{self.request.id}",+ json.dumps(result)+ )+return self.request.id