Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

added improvements

marcorasera54a9f8e1f86054
diff --git a/src/services/api/routes/retrieve.py b/src/services/api/routes/retrieve.py
index 7a02f93..ec2040e 100644
--- a/src/services/api/routes/retrieve.py
+++ b/src/services/api/routes/retrieve.py
@@ -159,20 +159,23 @@ async def get_entities(
)
@retrieve_router.get(path="/structured-data/types")
-async def get_structured_data_types():
+async def get_structured_data_types(
+ brain_id: str = "default",
+):
"""
Get all unique types from structured data.
"""
- return await get_structured_data_types_controller()
+ return await get_structured_data_types_controller(brain_id)
@retrieve_router.get(path="/structured-data/{id}")
async def get_structured_data_by_id(
id: str,
+ brain_id: str = "default",
):
"""
Get structured data by ID.
"""
- return await get_structured_data_by_id_controller(id)
+ return await get_structured_data_by_id_controller(id, brain_id)
@retrieve_router.get(path="/structured-data")
async def get_structured_data_list(
@@ -180,30 +183,33 @@ async def get_structured_data_list(
skip: int = 0,
types: Optional[str] = None,
query_text: Optional[str] = None,
+ brain_id: str = "default",
):
"""
Get a list of structured data.
"""
if types:
types = types.split(",")
- return await get_structured_data_list_controller(limit, skip, types, query_text)
+ return await get_structured_data_list_controller(limit, skip, types, query_text, brain_id)
@retrieve_router.get(path="/observations/labels")
-async def get_observation_labels():
+async def get_observation_labels(
+ brain_id: str = "default",
+):
"""
Get all unique labels from observations.
"""
- return await get_observation_labels_controller()
+ return await get_observation_labels_controller(brain_id)
@retrieve_router.get(path="/observations/{id}")
async def get_observation_by_id(
id: str,
+ brain_id: str = "default",
):
"""
Get observation by ID.
"""
- return await get_observation_by_id_controller(id)
-
+ return await get_observation_by_id_controller(id, brain_id)
@retrieve_router.get(path="/observations")
async def get_observations_list(
@@ -212,10 +218,11 @@ async def get_observations_list(
resource_id: Optional[str] = None,
labels: Optional[str] = None,
query_text: Optional[str] = None,
+ brain_id: str = "default",
):
"""
Get a list of observations.
"""
if labels:
labels = labels.split(",")
- return await get_observations_list_controller(limit, skip, resource_id, labels, query_text)
\ No newline at end of file
+ return await get_observations_list_controller(limit, skip, resource_id, labels, query_text, brain_id)
\ No newline at end of file
diff --git a/src/services/api/routes/tasks.py b/src/services/api/routes/tasks.py
index f8edbac..5c44e17 100644
--- a/src/services/api/routes/tasks.py
+++ b/src/services/api/routes/tasks.py
@@ -9,12 +9,11 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
"""
import json
from src.utils.logging import log
-from fastapi import APIRouter, HTTPException
+from fastapi import APIRouter
from src.services.kg_agent.main import cache_adapter
tasks_router = APIRouter(prefix="/tasks", tags=["tasks"])
-
@tasks_router.get("/{task_id}")
async def get_task(task_id: str, brain_id: str = "default"):
"""
@@ -22,18 +21,19 @@ async def get_task(task_id: str, brain_id: str = "default"):
"""
try:
str_result = cache_adapter.get(f"task:{task_id}", brain_id=brain_id)
-
if str_result is None:
- raise HTTPException(status_code=404, detail="Task not found")
-
+ return {
+ "task_id": task_id,
+ "status": "pending",
+ "result": {"message": "Task is still processing or not found"}
+ }
if isinstance(str_result, bytes):
result = json.loads(str_result.decode("utf-8"))
else:
result = json.loads(str_result)
-
return {
"task_id": task_id,
- "status": result.get("status", "pending"),
+ "status": result.get("status", "unknown"),
"result": result
}
except Exception as e:
@@ -41,5 +41,8 @@ async def get_task(task_id: str, brain_id: str = "default"):
return {
"task_id": task_id,
"status": "error",
- "result": {"error": str(e)}
+ "result": {
+ "error": str(e),
+ "error_type": type(e).__name__
+ }
}
\ No newline at end of file
diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.py
index a07cd34..9f5cb4e 100644
--- a/src/workers/tasks/ingestion.py
+++ b/src/workers/tasks/ingestion.py
@@ -43,108 +43,125 @@ def ingest_data(self, args: dict):
"""
Ingest data into the database.
"""
- payload = IngestionTaskArgs(**args)
+ try:
+ payload = IngestionTaskArgs(**args)
- payload.meta_keys = (
- {
- f"{k.replace(' ', '_').lower()}": v
- for k, v in payload.meta_keys.items()
- if v is not None
- }
- if payload.meta_keys
- else None
- )
+ payload.meta_keys = (
+ {
+ f"{k.replace(' ', '_').lower()}": v
+ for k, v in payload.meta_keys.items()
+ if v is not None
+ }
+ if payload.meta_keys
+ else None
+ )
+
+ payload.identification_params = (
+ {
+ f"{k.replace(' ', '_').lower()}": v
+ for k, v in payload.identification_params.items()
+ if v is not None
+ }
+ if payload.identification_params
+ else None
+ )
- payload.identification_params = (
- {
- f"{k.replace(' ', '_').lower()}": v
- for k, v in payload.identification_params.items()
- if v is not None
+ # ================================================
+ # --------------- Data Saving --------------------
+ # ================================================
+ text_chunk = data_adapter.save_text_chunk(
+ TextChunk(
+ text=(
+ payload.data.text_data
+ if payload.data.data_type == IngestionTaskDataType.TEXT.value
+ else json.dumps(payload.data.json_data)
+ ),
+ metadata=payload.meta_keys,
+ ),
+ brain_id=payload.brain_id,
+ )
+ text_chunk_vector = embeddings_adapter.embed_text(text_chunk.text)
+ text_chunk_vector.metadata = {
+ **payload.meta_keys,
+ "resource_id": text_chunk.id,
}
- if payload.identification_params
- else None
- )
+ vector_store_adapter.add_vectors(
+ [text_chunk_vector],
+ "data",
+ )
- # ================================================
- # --------------- Data Saving --------------------
- # ================================================
- text_chunk = data_adapter.save_text_chunk(
- TextChunk(
+ # ================================================
+ # --------------- Observations -------------------
+ # ================================================
+ observations = observations_agent.observe(
text=(
payload.data.text_data
if payload.data.data_type == IngestionTaskDataType.TEXT.value
else json.dumps(payload.data.json_data)
),
- metadata=payload.meta_keys,
- ),
- brain_id=payload.brain_id,
- )
- text_chunk_vector = embeddings_adapter.embed_text(text_chunk.text)
- text_chunk_vector.metadata = {
- **payload.meta_keys,
- "resource_id": text_chunk.id,
- }
- vector_store_adapter.add_vectors(
- [text_chunk_vector],
- "data",
- )
+ observate_for=payload.observate_for,
+ )
- # ================================================
- # --------------- Observations -------------------
- # ================================================
- observations = observations_agent.observe(
- text=(
- payload.data.text_data
- if payload.data.data_type == IngestionTaskDataType.TEXT.value
- else json.dumps(payload.data.json_data)
- ),
- observate_for=payload.observate_for,
- )
+ data_adapter.save_observations(
+ [
+ Observation(
+ text=observation,
+ metadata=payload.meta_keys,
+ resource_id=text_chunk.id,
+ )
+ for observation in observations
+ ]
+ )
- data_adapter.save_observations(
- [
- Observation(
- text=observation,
- metadata=payload.meta_keys,
- resource_id=text_chunk.id,
- )
- for observation in observations
- ]
- )
+ # ================================================
+ # ------------ Triplet Extraction ----------------
+ # ================================================
+ information = f"""
+ {payload.data.text_data
+ if payload.data.data_type == IngestionTaskDataType.TEXT.value
+ else json.dumps(payload.data.json_data)}
+
+ Annotations:
+ {observations}
+ """
- # ================================================
- # ------------ Triplet Extraction ----------------
- # ================================================
- information = f"""
- {payload.data.text_data
- if payload.data.data_type == IngestionTaskDataType.TEXT.value
- else json.dumps(payload.data.json_data)}
-
- Annotations:
- {observations}
- """
+ kg_agent.update_kg(
+ information=information,
+ metadata={**payload.meta_keys, "resource_id": text_chunk.id},
+ identification_params=payload.identification_params,
+ preferred_entities=payload.preferred_extraction_entities,
+ brain_id=payload.brain_id,
+ )
- kg_agent.update_kg(
- information=information,
- metadata={**payload.meta_keys, "resource_id": text_chunk.id},
- identification_params=payload.identification_params,
- preferred_entities=payload.preferred_extraction_entities,
- brain_id=payload.brain_id,
- )
+ result = {
+ "status": "completed",
+ "task_id": self.request.id,
+ "text_chunk_id": text_chunk.id,
+ "observations_count": len(observations)
+ }
+
+ cache_adapter.set(
+ f"task:{self.request.id}",
+ json.dumps(result),
+ expires_in=3600
+ )
- result = {
- "status": "completed",
- "task_id": self.request.id,
- "text_chunk_id": text_chunk.id,
- "observations_count": len(observations)
- }
+ return self.request.id
- cache_adapter.set(
- f"task:{self.request.id}",
- json.dumps(result)
- )
-
- return self.request.id
+ except Exception as e:
+ error_result = {
+ "status": "failed",
+ "task_id": self.request.id,
+ "error": str(e),
+ }
+
+ cache_adapter.set(
+ f"task:{self.request.id}",
+ json.dumps(error_result),
+ expires_in=3600
+ )
+
+ raise
@ingestion_app.task(bind=True)
@@ -152,154 +169,171 @@ def ingest_structured_data(self, args: dict):
"""
Ingest structured data into the database.
"""
- payload = IngestionStructuredRequestBody(**args)
+ try:
+ payload = IngestionStructuredRequestBody(**args)
- for element in payload.data:
- uuid = str(uuid4())
- node = graph_adapter.add_nodes(
- [
- Node(
- uuid=uuid,
- labels=element.types,
- name=element.identification_params.name,
- properties={
- **(element.json_data if element.json_data else {}),
- **(
- element.identification_params.model_dump(mode="json")
- if element.identification_params
- else {}
- ),
- },
- )
- ],
- brain_id=payload.brain_id,
- identification_params=element.identification_params.model_dump(mode="json"),
- metadata=element.textual_data,
- )[0]
-
- # Saving the node properties updated change
- kg_changes = KGChanges(
- type=KGChangesType.NODE_PROPERTIES_UPDATED,
- change=KGChangeLogNodePropertiesUpdated(
- node=PartialNode(
- **node.model_dump(mode="json"),
- ),
- properties=[
- KGChangeLogPredicateUpdatedProperty(
- property=property,
- previous_value=None,
- new_value=node.properties[property],
+ for element in payload.data:
+ uuid = str(uuid4())
+ node = graph_adapter.add_nodes(
+ [
+ Node(
+ uuid=uuid,
+ labels=element.types,
+ name=element.identification_params.name,
+ properties={
+ **(element.json_data if element.json_data else {}),
+ **(
+ element.identification_params.model_dump(mode="json")
+ if element.identification_params
+ else {}
+ ),
+ },
)
- for property in node.properties
],
- ),
- )
- data_adapter.save_kg_changes(kg_changes, brain_id=payload.brain_id)
-
- vector = embeddings_adapter.embed_text(node.name)
- vector.id = node.uuid
- vector.metadata = {
- "labels": node.labels,
- "name": node.name,
- "uuid": node.uuid,
- }
- vector_store_adapter.add_vectors(vectors=[vector], store="nodes")
+ brain_id=payload.brain_id,
+ identification_params=element.identification_params.model_dump(mode="json"),
+ metadata=element.textual_data,
+ )[0]
- data_adapter.save_structured_data(
- StructuredData(
- id=uuid,
- data=element.json_data,
- types=element.types,
- metadata={
- "identification_params": element.identification_params,
- "textual_data": element.textual_data,
- "labels": element.types,
- "name": element.identification_params.name,
- },
- ),
- brain_id=payload.brain_id,
- )
- vector = embeddings_adapter.embed_text(
- "\n".join(
- [
- (
- v
- if isinstance(v, str)
- else (", ".join(v) if isinstance(v, list) else str(v))
- )
- for v in element.textual_data.values()
- ]
+ # Saving the node properties updated change
+ kg_changes = KGChanges(
+ type=KGChangesType.NODE_PROPERTIES_UPDATED,
+ change=KGChangeLogNodePropertiesUpdated(
+ node=PartialNode(
+ **node.model_dump(mode="json"),
+ ),
+ properties=[
+ KGChangeLogPredicateUpdatedProperty(
+ property=property,
+ previous_value=None,
+ new_value=node.properties[property],
+ )
+ for property in node.properties
+ ],
+ ),
)
- + "\n"
- + (", ".join(node.labels) if isinstance(node.labels, list) else node.labels)
- + "\n"
- + node.name,
- )
- vector.id = node.uuid
- vector.metadata = {
- "labels": node.labels,
- "name": node.name,
- "uuid": node.uuid,
- }
- vector_store_adapter.add_vectors(vectors=[vector], store="data")
+ data_adapter.save_kg_changes(kg_changes, brain_id=payload.brain_id)
- text_content = "\n".join(
- [
- (
- f"{k}: {v}"
- if isinstance(v, str)
- else (f"{k}: {', '.join(v) if isinstance(v, list) else str(v)}")
+ vector = embeddings_adapter.embed_text(node.name)
+ vector.id = node.uuid
+ vector.metadata = {
+ "labels": node.labels,
+ "name": node.name,
+ "uuid": node.uuid,
+ }
+ vector_store_adapter.add_vectors(vectors=[vector], store="nodes")
+
+ data_adapter.save_structured_data(
+ StructuredData(
+ id=uuid,
+ data=element.json_data,
+ types=element.types,
+ metadata={
+ "identification_params": element.identification_params,
+ "textual_data": element.textual_data,
+ "labels": element.types,
+ "name": element.identification_params.name,
+ },
+ ),
+ brain_id=payload.brain_id,
+ )
+ vector = embeddings_adapter.embed_text(
+ "\n".join(
+ [
+ (
+ v
+ if isinstance(v, str)
+ else (", ".join(v) if isinstance(v, list) else str(v))
+ )
+ for v in element.textual_data.values()
+ ]
)
- for k, v in element.textual_data.items()
- ]
- )
- observations = observations_agent.observe(
- text=text_content,
- observate_for=payload.observate_for,
- )
- for obs in observations:
- obs_vector = embeddings_adapter.embed_text(obs)
- obs_vector.id = str(uuid4())
- obs_vector.metadata = {
- "resource_id": node.uuid,
+ + "\n"
+ + (", ".join(node.labels) if isinstance(node.labels, list) else node.labels)
+ + "\n"
+ + node.name,
+ )
+ vector.id = node.uuid
+ vector.metadata = {
"labels": node.labels,
"name": node.name,
+ "uuid": node.uuid,
}
- vector_store_adapter.add_vectors(vectors=[obs_vector], store="observations")
- if len(observations) > 0:
- data_adapter.save_observations(
+ vector_store_adapter.add_vectors(vectors=[vector], store="data")
+
+ text_content = "\n".join(
[
- Observation(
- id=node.uuid,
- text=obs,
- metadata={
- "labels": node.labels,
- "name": node.name,
- },
- resource_id=node.uuid,
+ (
+ f"{k}: {v}"
+ if isinstance(v, str)
+ else (f"{k}: {', '.join(v) if isinstance(v, list) else str(v)}")
)
- for obs in observations
- ],
+ for k, v in element.textual_data.items()
+ ]
+ )
+ observations = observations_agent.observe(
+ text=text_content,
+ observate_for=payload.observate_for,
+ )
+ for obs in observations:
+ obs_vector = embeddings_adapter.embed_text(obs)
+ obs_vector.id = str(uuid4())
+ obs_vector.metadata = {
+ "resource_id": node.uuid,
+ "labels": node.labels,
+ "name": node.name,
+ }
+ vector_store_adapter.add_vectors(vectors=[obs_vector], store="observations")
+ if len(observations) > 0:
+ data_adapter.save_observations(
+ [
+ Observation(
+ id=node.uuid,
+ text=obs,
+ metadata={
+ "labels": node.labels,
+ "name": node.name,
+ },
+ resource_id=node.uuid,
+ )
+ for obs in observations
+ ],
+ brain_id=payload.brain_id,
+ )
+
+ # TODO: + vectorize the predicates and new nodes + save changelog&vectors
+ kg_agent.structured_update_kg(
+ main_node=node,
+ textual_data=element.textual_data,
+ identification_params=element.identification_params.model_dump(mode="json"),
brain_id=payload.brain_id,
)
- # TODO: + vectorize the predicates and new nodes + save changelog&vectors
- kg_agent.structured_update_kg(
- main_node=node,
- textual_data=element.textual_data,
- identification_params=element.identification_params.model_dump(mode="json"),
- brain_id=payload.brain_id,
+ result = {
+ "status": "completed",
+ "task_id": self.request.id,
+ "ingested_count": len(payload.data)
+ }
+
+ cache_adapter.set(
+ f"task:{self.request.id}",
+ json.dumps(result),
+ expires_in=3600
)
- result = {
- "status": "completed",
- "task_id": self.request.id,
- "ingested_count": len(payload.data)
- }
+ return self.request.id
- cache_adapter.set(
- f"task:{self.request.id}",
- json.dumps(result)
- )
-
- return self.request.id
+ except Exception as e:
+ error_result = {
+ "status": "failed",
+ "task_id": self.request.id,
+ "error": str(e),
+ }
+
+ cache_adapter.set(
+ f"task:{self.request.id}",
+ json.dumps(error_result),
+ expires_in=3600
+ )
+
+ raise
\ No newline at end of file