Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

implemented tasks, observations, structured_data logic

marcorasera54c2382fe08b7e
diff --git a/.gitignore b/.gitignore
index 1899141..2a0da12 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,4 +6,5 @@ __pycache__
dist
common.egg-info
.venv
-.env.development
\ No newline at end of file
+.env.development
+venv/
\ No newline at end of file
diff --git a/src/adapters/data.py b/src/adapters/data.py
index 9fd406f..c09edb1 100644
--- a/src/adapters/data.py
+++ b/src/adapters/data.py
@@ -3,7 +3,7 @@ File: /data.py
Created Date: Sunday October 19th 2025
Author: Christian Nonis <[email protected]>
-----
-Last Modified: Sunday October 19th 2025 9:00:42 am
+Last Modified: Saturday December 13th 2025
Modified By: the developer formerly known as Christian Nonis at <[email protected]>
-----
"""
@@ -90,3 +90,53 @@ class DataAdapter:
Save a KG changes to the data client.
"""
return self.data.save_kg_changes(kg_changes, brain_id)
+
+ def get_structured_data_by_id(
+ self, id: str, brain_id: str = "default"
+ ) -> StructuredData:
+ """
+ Get structured data by ID.
+ """
+ return self.data.get_structured_data_by_id(id, brain_id)
+
+ def get_structured_data_list(
+ self, brain_id: str = "default", limit: int = 10, skip: int = 0, types: list[str] = None, query_text: str = None
+ ) -> list[StructuredData]:
+ """
+ Get a list of structured data.
+ """
+ return self.data.get_structured_data_list(brain_id, limit, skip, types, query_text)
+
+ def get_structured_data_types(self, brain_id: str = "default") -> list[str]:
+ """
+ Get all unique types from structured data.
+ """
+ return self.data.get_structured_data_types(brain_id)
+
+ def get_observation_by_id(
+ self, id: str, brain_id: str = "default"
+ ) -> Observation:
+ """
+ Get observation by ID.
+ """
+ return self.data.get_observation_by_id(id, brain_id)
+
+ def get_observations_list(
+ self,
+ brain_id: str = "default",
+ limit: int = 10,
+ skip: int = 0,
+ resource_id: str = None,
+ labels: list[str] = None,
+ query_text: str = None
+ ) -> list[Observation]:
+ """
+ Get a list of observations.
+ """
+ return self.data.get_observations_list(brain_id, limit, skip, resource_id, labels, query_text)
+
+ def get_observation_labels(self, brain_id: str = "default") -> list[str]:
+ """
+ Get all unique labels from observations.
+ """
+ return self.data.get_observation_labels(brain_id)
\ No newline at end of file
diff --git a/src/adapters/interfaces/data.py b/src/adapters/interfaces/data.py
index 24717db..fd741eb 100644
--- a/src/adapters/interfaces/data.py
+++ b/src/adapters/interfaces/data.py
@@ -3,7 +3,7 @@ File: /data.py
Created Date: Sunday October 19th 2025
Author: Christian Nonis <[email protected]>
-----
-Last Modified: Sunday October 19th 2025 9:01:13 am
+Last Modified: Saturday December 13th 2025
Modified By: the developer formerly known as Christian Nonis at <[email protected]>
-----
"""
@@ -98,3 +98,55 @@ class DataClient(ABC):
Save a KG changes to the data client.
"""
raise NotImplementedError("save_kg_changes method not implemented")
+
+ @abstractmethod
+ def get_structured_data_by_id(self, id: str, brain_id: str) -> StructuredData:
+ """
+ Get structured data by ID.
+ """
+ raise NotImplementedError("get_structured_data_by_id method not implemented")
+
+ @abstractmethod
+ def get_structured_data_list(
+ self, brain_id: str, limit: int = 10, skip: int = 0, types: list[str] = None, query_text: str = None
+ ) -> list[StructuredData]:
+ """
+ Get a list of structured data.
+ """
+ raise NotImplementedError("get_structured_data_list method not implemented")
+
+ @abstractmethod
+ def get_structured_data_types(self, brain_id: str) -> list[str]:
+ """
+ Get all unique types from structured data.
+ """
+ raise NotImplementedError("get_structured_data_types method not implemented")
+
+ @abstractmethod
+ def get_observation_by_id(self, id: str, brain_id: str) -> Observation:
+ """
+ Get observation by ID.
+ """
+ raise NotImplementedError("get_observation_by_id method not implemented")
+
+ @abstractmethod
+ def get_observations_list(
+ self,
+ brain_id: str,
+ limit: int = 10,
+ skip: int = 0,
+ resource_id: str = None,
+ labels: list[str] = None,
+ query_text: str = None
+ ) -> list[Observation]:
+ """
+ Get a list of observations.
+ """
+ raise NotImplementedError("get_observations_list method not implemented")
+
+ @abstractmethod
+ def get_observation_labels(self, brain_id: str) -> list[str]:
+ """
+ Get all unique labels from observations.
+ """
+ raise NotImplementedError("get_observation_labels method not implemented")
\ No newline at end of file
diff --git a/src/lib/mongo/client.py b/src/lib/mongo/client.py
index 124e155..a7a985d 100644
--- a/src/lib/mongo/client.py
+++ b/src/lib/mongo/client.py
@@ -3,7 +3,7 @@ File: /client.py
Created Date: Saturday October 25th 2025
Author: Christian Nonis <[email protected]>
-----
-Last Modified: Saturday October 25th 2025 11:55:47 am
+Last Modified: Saturday December 13th 2025
Modified By: the developer formerly known as Christian Nonis at <[email protected]>
-----
"""
@@ -168,5 +168,117 @@ class MongoClient(DataClient):
collection.insert_one(kg_changes.model_dump(mode="json"))
return kg_changes
+ def get_structured_data_by_id(
+ self, id: str, brain_id: str
+ ) -> StructuredData:
+ collection = self.get_collection("structured_data", database=brain_id)
+ result = collection.find_one({"id": id})
+ if not result:
+ return None
+ return StructuredData(
+ id=result["id"],
+ data=result["data"],
+ types=result["types"],
+ metadata=result.get("metadata", None),
+ inserted_at=result.get("inserted_at", None),
+ )
+
+ def get_structured_data_list(
+ self, brain_id: str, limit: int = 10, skip: int = 0, types: list[str] = None, query_text: str = None
+ ) -> list[StructuredData]:
+ collection = self.get_collection("structured_data", database=brain_id)
+ query = {}
+ if types:
+ query["types"] = {"$in": types}
+
+ if query_text:
+ query["$or"] = [
+ {"data": {"$regex": query_text, "$options": "i"}},
+ {"types": {"$regex": query_text, "$options": "i"}},
+ {"metadata": {"$regex": query_text, "$options": "i"}}
+ ]
+
+ results = collection.find(query).skip(skip).limit(limit)
+ return [
+ StructuredData(
+ id=result["id"],
+ data=result["data"],
+ types=result["types"],
+ metadata=result.get("metadata", None),
+ inserted_at=result.get("inserted_at", None),
+ )
+ for result in results
+ ]
+
+ def get_structured_data_types(self, brain_id: str) -> list[str]:
+ collection = self.get_collection("structured_data", database=brain_id)
+ pipeline = [
+ {"$unwind": "$types"},
+ {"$group": {"_id": "$types"}},
+ {"$sort": {"_id": 1}}
+ ]
+ results = collection.aggregate(pipeline)
+ return [result["_id"] for result in results]
+
+ def get_observation_by_id(
+ self, id: str, brain_id: str
+ ) -> Observation:
+ collection = self.get_collection("observations", database=brain_id)
+ result = collection.find_one({"id": id})
+ if not result:
+ return None
+ return Observation(
+ id=result["id"],
+ text=result["text"],
+ metadata=result.get("metadata", None),
+ resource_id=result["resource_id"],
+ inserted_at=result.get("inserted_at", None),
+ )
+
+ def get_observations_list(
+ self,
+ brain_id: str,
+ limit: int = 10,
+ skip: int = 0,
+ resource_id: str = None,
+ labels: list[str] = None,
+ query_text: str = None
+ ) -> list[Observation]:
+ collection = self.get_collection("observations", database=brain_id)
+ query = {}
+ if resource_id:
+ query["resource_id"] = resource_id
+ if labels:
+ query["metadata.labels"] = {"$in": labels}
+
+ if query_text:
+ query["$or"] = [
+ {"text": {"$regex": query_text, "$options": "i"}},
+ {"resource_id": {"$regex": query_text, "$options": "i"}},
+ {"metadata": {"$regex": query_text, "$options": "i"}}
+ ]
+
+ results = collection.find(query).skip(skip).limit(limit)
+ return [
+ Observation(
+ id=result["id"],
+ text=result["text"],
+ metadata=result.get("metadata", None),
+ resource_id=result["resource_id"],
+ inserted_at=result.get("inserted_at", None),
+ )
+ for result in results
+ ]
+
+ def get_observation_labels(self, brain_id: str) -> list[str]:
+ collection = self.get_collection("observations", database=brain_id)
+ pipeline = [
+ {"$match": {"metadata.labels": {"$exists": True}}},
+ {"$unwind": "$metadata.labels"},
+ {"$group": {"_id": "$metadata.labels"}},
+ {"$sort": {"_id": 1}}
+ ]
+ results = collection.aggregate(pipeline)
+ return [result["_id"] for result in results]
_mongo_client = MongoClient()
diff --git a/src/services/api/app.py b/src/services/api/app.py
index 4679af1..7cf6e9a 100644
--- a/src/services/api/app.py
+++ b/src/services/api/app.py
@@ -20,6 +20,7 @@ from src.services.api.routes.ingest import ingest_router
from src.services.api.routes.retrieve import retrieve_router
from src.services.api.routes.meta import meta_router
from src.services.api.routes.system import system_router
+from src.services.api.routes.tasks import tasks_router
app = FastAPI(debug=os.getenv("ENV") == "development")
@@ -38,7 +39,7 @@ app.include_router(ingest_router)
app.include_router(retrieve_router)
app.include_router(meta_router)
app.include_router(system_router)
-
+app.include_router(tasks_router)
@app.get("/")
async def root():
diff --git a/src/services/api/controllers/observations.py b/src/services/api/controllers/observations.py
new file mode 100644
index 0000000..77115c6
--- /dev/null
+++ b/src/services/api/controllers/observations.py
@@ -0,0 +1,83 @@
+"""
+File: /observations.py
+Created Date: Saturday December 6th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Saturday December 13th 2025
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
+
+import asyncio
+from typing import Optional
+from fastapi import HTTPException
+from starlette.responses import JSONResponse
+from src.services.data.main import data_adapter
+
+
+async def get_observation_by_id(
+ id: str, brain_id: str = "default"
+):
+ """
+ Get observation by ID.
+ """
+ def _get_observation():
+ observation = data_adapter.get_observation_by_id(id, brain_id)
+ if not observation:
+ raise HTTPException(status_code=404, detail="Observation not found")
+ return observation
+
+ result = await asyncio.to_thread(_get_observation)
+
+ return JSONResponse(
+ content={
+ "message": "Observation retrieved successfully",
+ "observation": result.model_dump(mode="json")
+ }
+ )
+
+
+async def get_observations_list(
+ limit: int = 10,
+ skip: int = 0,
+ resource_id: Optional[str] = None,
+ labels: Optional[list[str]] = None,
+ query_text: Optional[str] = None,
+ brain_id: str = "default",
+):
+ """
+ Get a list of observations.
+ """
+ def _get_list():
+ return data_adapter.get_observations_list(
+ brain_id, limit, skip, resource_id, labels, query_text
+ )
+
+ results = await asyncio.to_thread(_get_list)
+
+ return JSONResponse(
+ content={
+ "message": "Observations list retrieved successfully",
+ "observations": [r.model_dump(mode="json") for r in results],
+ "count": len(results)
+ }
+ )
+
+async def get_observation_labels(
+ brain_id: str = "default",
+):
+ """
+ Get all unique labels from observations.
+ """
+ def _get_labels():
+ return data_adapter.get_observation_labels(brain_id)
+
+ results = await asyncio.to_thread(_get_labels)
+
+ return JSONResponse(
+ content={
+ "message": "Observation labels retrieved successfully",
+ "labels": results,
+ "count": len(results)
+ }
+ )
\ No newline at end of file
diff --git a/src/services/api/controllers/structured_data.py b/src/services/api/controllers/structured_data.py
new file mode 100644
index 0000000..5c65913
--- /dev/null
+++ b/src/services/api/controllers/structured_data.py
@@ -0,0 +1,79 @@
+"""
+File: /structured_data.py
+Created Date: Saturday December 6th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Saturday December 13th 2025
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
+
+import asyncio
+from typing import Optional
+from fastapi import HTTPException
+from starlette.responses import JSONResponse
+from src.services.data.main import data_adapter
+
+async def get_structured_data_by_id(
+ id: str, brain_id: str = "default"
+):
+ """
+ Get structured data by ID.
+ """
+ def _get_data():
+ data = data_adapter.get_structured_data_by_id(id, brain_id)
+ if not data:
+ raise HTTPException(status_code=404, detail="Structured data not found")
+ return data
+
+ result = await asyncio.to_thread(_get_data)
+
+ return JSONResponse(
+ content={
+ "message": "Structured data retrieved successfully",
+ "data": result.model_dump(mode="json")
+ }
+ )
+
+
+async def get_structured_data_list(
+ limit: int = 10,
+ skip: int = 0,
+ types: Optional[list[str]] = None,
+ query_text: Optional[str] = None,
+ brain_id: str = "default",
+):
+ """
+ Get a list of structured data.
+ """
+ def _get_list():
+ return data_adapter.get_structured_data_list(brain_id, limit, skip, types, query_text)
+
+ results = await asyncio.to_thread(_get_list)
+
+ return JSONResponse(
+ content={
+ "message": "Structured data list retrieved successfully",
+ "data": [r.model_dump(mode="json") for r in results],
+ "count": len(results)
+ }
+ )
+
+async def get_structured_data_types(
+ brain_id: str = "default",
+):
+ """
+ Get all unique types from structured data.
+ """
+ def _get_types():
+ return data_adapter.get_structured_data_types(brain_id)
+
+ results = await asyncio.to_thread(_get_types)
+
+ return JSONResponse(
+ content={
+ "message": "Structured data types retrieved successfully",
+ "types": results,
+ "count": len(results)
+ }
+ )
\ No newline at end of file
diff --git a/src/services/api/routes/retrieve.py b/src/services/api/routes/retrieve.py
index bb3112d..7a02f93 100644
--- a/src/services/api/routes/retrieve.py
+++ b/src/services/api/routes/retrieve.py
@@ -3,7 +3,7 @@ File: /retrieve.py
Created Date: Saturday October 25th 2025
Author: Christian Nonis <[email protected]>
-----
-Last Modified: Saturday October 25th 2025 12:32:21 pm
+Last Modified: Saturday December 13th 2025
Modified By: the developer formerly known as Christian Nonis at <[email protected]>
-----
"""
@@ -26,6 +26,16 @@ from src.services.api.controllers.retrieve import (
from src.services.api.controllers.retrieve import (
retrieve_data as retrieve_data_controller,
)
+from src.services.api.controllers.structured_data import (
+ get_structured_data_by_id as get_structured_data_by_id_controller,
+ get_structured_data_list as get_structured_data_list_controller,
+ get_structured_data_types as get_structured_data_types_controller,
+)
+from src.services.api.controllers.observations import (
+ get_observation_by_id as get_observation_by_id_controller,
+ get_observations_list as get_observations_list_controller,
+ get_observation_labels as get_observation_labels_controller,
+)
retrieve_router = APIRouter(prefix="/retrieve", tags=["retrieve"])
@@ -147,3 +157,65 @@ async def get_entities(
return await retrieve_get_entities_controller(
limit, skip, node_labels, query_text, brain_id
)
+
+@retrieve_router.get(path="/structured-data/types")
+async def get_structured_data_types():
+ """
+ Get all unique types from structured data.
+ """
+ return await get_structured_data_types_controller()
+
+@retrieve_router.get(path="/structured-data/{id}")
+async def get_structured_data_by_id(
+ id: str,
+):
+ """
+ Get structured data by ID.
+ """
+ return await get_structured_data_by_id_controller(id)
+
+@retrieve_router.get(path="/structured-data")
+async def get_structured_data_list(
+ limit: int = 10,
+ skip: int = 0,
+ types: Optional[str] = None,
+ query_text: Optional[str] = None,
+):
+ """
+ Get a list of structured data.
+ """
+ if types:
+ types = types.split(",")
+ return await get_structured_data_list_controller(limit, skip, types, query_text)
+
+@retrieve_router.get(path="/observations/labels")
+async def get_observation_labels():
+ """
+ Get all unique labels from observations.
+ """
+ return await get_observation_labels_controller()
+
+@retrieve_router.get(path="/observations/{id}")
+async def get_observation_by_id(
+ id: str,
+):
+ """
+ Get observation by ID.
+ """
+ return await get_observation_by_id_controller(id)
+
+
+@retrieve_router.get(path="/observations")
+async def get_observations_list(
+ limit: int = 10,
+ skip: int = 0,
+ resource_id: Optional[str] = None,
+ labels: Optional[str] = None,
+ query_text: Optional[str] = None,
+):
+ """
+ Get a list of observations.
+ """
+ if labels:
+ labels = labels.split(",")
+ return await get_observations_list_controller(limit, skip, resource_id, labels, query_text)
\ No newline at end of file
diff --git a/src/services/api/routes/tasks.py b/src/services/api/routes/tasks.py
new file mode 100644
index 0000000..f8edbac
--- /dev/null
+++ b/src/services/api/routes/tasks.py
@@ -0,0 +1,45 @@
+"""
+File: /tasks.py
+Created Date: Saturday December 13th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Saturday December 13th 2025
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
+import json
+from src.utils.logging import log
+from fastapi import APIRouter, HTTPException
+from src.services.kg_agent.main import cache_adapter
+
+tasks_router = APIRouter(prefix="/tasks", tags=["tasks"])
+
+
+@tasks_router.get("/{task_id}")
+async def get_task(task_id: str, brain_id: str = "default"):
+ """
+ Get the result of a task by its ID.
+ """
+ try:
+ str_result = cache_adapter.get(f"task:{task_id}", brain_id=brain_id)
+
+ if str_result is None:
+ raise HTTPException(status_code=404, detail="Task not found")
+
+ if isinstance(str_result, bytes):
+ result = json.loads(str_result.decode("utf-8"))
+ else:
+ result = json.loads(str_result)
+
+ return {
+ "task_id": task_id,
+ "status": result.get("status", "pending"),
+ "result": result
+ }
+ except Exception as e:
+ log(f"Error in get_task: {type(e).__name__}: {str(e)}")
+ return {
+ "task_id": task_id,
+ "status": "error",
+ "result": {"error": str(e)}
+ }
\ No newline at end of file
diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.py
index aad08a1..a07cd34 100644
--- a/src/workers/tasks/ingestion.py
+++ b/src/workers/tasks/ingestion.py
@@ -3,7 +3,7 @@ File: /ingestion.py
Created Date: Sunday October 19th 2025
Author: Christian Nonis <[email protected]>
-----
-Last Modified: Sunday October 19th 2025 12:14:21 pm
+Last Modified: Saturday December 13th 2025
Modified By: the developer formerly known as Christian Nonis at <[email protected]>
-----
"""
@@ -35,7 +35,8 @@ from src.constants.tasks.ingestion import (
IngestionTaskArgs,
IngestionTaskDataType,
)
-
+import json
+from src.services.kg_agent.main import cache_adapter
@ingestion_app.task(bind=True)
def ingest_data(self, args: dict):
@@ -131,6 +132,18 @@ def ingest_data(self, args: dict):
brain_id=payload.brain_id,
)
+ result = {
+ "status": "completed",
+ "task_id": self.request.id,
+ "text_chunk_id": text_chunk.id,
+ "observations_count": len(observations)
+ }
+
+ cache_adapter.set(
+ f"task:{self.request.id}",
+ json.dumps(result)
+ )
+
return self.request.id
@@ -278,4 +291,15 @@ def ingest_structured_data(self, args: dict):
brain_id=payload.brain_id,
)
+ result = {
+ "status": "completed",
+ "task_id": self.request.id,
+ "ingested_count": len(payload.data)
+ }
+
+ cache_adapter.set(
+ f"task:{self.request.id}",
+ json.dumps(result)
+ )
+
return self.request.id