BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
feat(read/write)
Christian422e520ecd6e
diff --git a/Makefile b/Makefileindex a164a18..6720a61 100644--- a/Makefile+++ b/Makefile@@ -35,6 +35,15 @@ stop-neo4j:delete-neo4j-volumes:docker compose -f src/lib/neo4j/docker-compose.yaml down -v --remove-orphans+start-mongo:+ docker compose -f src/lib/mongo/docker-compose.yaml up -d++stop-mongo:+ docker compose -f src/lib/mongo/docker-compose.yaml down++delete-mongo-volumes:+ docker compose -f src/lib/mongo/docker-compose.yaml down -v --remove-orphans+start-api:poetry run uvicorn src.services.api.app:app --host 0.0.0.0 --port 8000@@ -52,6 +61,7 @@ start-all:make start-rabbitmq &make start-redis &make start-neo4j &+ make start-mongo &make start-api &@bash -c "poetry run celery -A src.workers.app worker --loglevel=info"@@ -60,6 +70,7 @@ stop-all:make stop-rabbitmqmake stop-redismake stop-neo4j+ make stop-mongomake stop-apipkill -f celery@@ -67,4 +78,5 @@ clear-all-volumes:make delete-milvus-volumesmake delete-rabbitmq-volumesmake delete-redis-volumes- make delete-neo4j-volumes\ No newline at end of file+ make delete-neo4j-volumes+ make delete-mongo-volumes\ No newline at end of filediff --git a/src/adapters/data.py b/src/adapters/data.pyindex 708d5db..1acc111 100644--- a/src/adapters/data.py+++ b/src/adapters/data.py@@ -7,3 +7,48 @@ Last Modified: Sunday October 19th 2025 9:00:42 amModified By: the developer formerly known as Christian Nonis at <[email protected]>-----"""++from typing import List, Tuple+from src.adapters.interfaces.data import DataClient, SearchResult+from src.constants.data import Observation, TextChunk+++class DataAdapter:+ """+ Data adapter.+ """++ def __init__(self):+ self.data = None++ def add_client(self, client: DataClient) -> None:+ """+ Add a data client to the adapter.+ """+ self.data = client++ def save_text_chunk(self, text_chunk: TextChunk) -> TextChunk:+ """+ Save a text chunk to the data client.+ """+ return self.data.save_text_chunk(text_chunk)++ def save_observations(self, observations: List[Observation]) -> Observation:+ """+ Save a list of observations to the data client.+ """+ return self.data.save_observations(observations)++ def search(self, text: str) -> SearchResult:+ """+ Search data by text and return a list of text chunks and observations.+ """+ return self.data.search(text)++ def get_text_chunks_by_ids(+ self, ids: List[str], with_observations: bool+ ) -> Tuple[List[TextChunk], List[Observation]]:+ """+ Get data by their IDs.+ """+ return self.data.get_text_chunks_by_ids(ids, with_observations)diff --git a/src/adapters/graph.py b/src/adapters/graph.pyindex 7afb768..ed16ae6 100644--- a/src/adapters/graph.py+++ b/src/adapters/graph.py@@ -96,13 +96,24 @@ class GraphAdapter:with_relationships: Optional[bool] = False,relationships_depth: Optional[int] = 1,relationships_type: Optional[list[str]] = None,+ preferred_labels: Optional[list[str]] = None,) -> list[Node]:"""- Get nodes by their UUIDs.+ Get nodes by their UUIDs with optional relationships and preferred labels."""return self.graph.get_nodes_by_uuid(- uuids, with_relationships, relationships_depth, relationships_type+ uuids,+ with_relationships,+ relationships_depth,+ relationships_type,+ preferred_labels,)+ def get_graph_entities(self) -> list[str]:+ """+ Get the entities of the graph.+ """+ return self.graph.get_graph_entities()+_graph_adapter = GraphAdapter()diff --git a/src/adapters/interfaces/data.py b/src/adapters/interfaces/data.pyindex 7492275..6f426f0 100644--- a/src/adapters/interfaces/data.py+++ b/src/adapters/interfaces/data.py@@ -7,3 +7,55 @@ Last Modified: Sunday October 19th 2025 9:01:13 amModified By: the developer formerly known as Christian Nonis at <[email protected]>-----"""++from abc import ABC, abstractmethod+from typing import List, Tuple++from pydantic import BaseModel++from src.constants.data import Observation, TextChunk+++class SearchResult(BaseModel):+ """+ Search result model.+ """++ text_chunks: List[TextChunk]+ observations: List[Observation]+++class DataClient(ABC):+ """+ Abstract base class for data clients.+ """++ @abstractmethod+ def save_text_chunk(self, text_chunk: TextChunk) -> TextChunk:+ """+ Save data to the data client.+ """+ raise NotImplementedError("save method not implemented")++ @abstractmethod+ def save_observations(self, observations: List[Observation]) -> Observation:+ """+ Save a list of observations to the data client.+ """+ raise NotImplementedError("save_observations method not implemented")++ @abstractmethod+ def search(self, text: str) -> SearchResult:+ """+ Search data by text and return a list of text chunks and observations.+ """+ raise NotImplementedError("search method not implemented")++ @abstractmethod+ def get_text_chunks_by_ids(+ self, ids: List[str], with_observations: bool+ ) -> Tuple[List[TextChunk], List[Observation]]:+ """+ Get data by their IDs.+ """+ raise NotImplementedError("get_by_ids method not implemented")diff --git a/src/adapters/interfaces/graph.py b/src/adapters/interfaces/graph.pyindex 3fa64cd..3ae7e6e 100644--- a/src/adapters/interfaces/graph.py+++ b/src/adapters/interfaces/graph.py@@ -90,8 +90,16 @@ class GraphClient(ABC):with_relationships: Optional[bool] = False,relationships_depth: Optional[int] = 1,relationships_type: Optional[list[str]] = None,+ preferred_labels: Optional[list[str]] = None,) -> list[Node]:"""- Get nodes by their UUIDs with optional relationships.+ Get nodes by their UUIDs with optional relationships and preferred labels."""raise NotImplementedError("get_nodes_by_uuid method not implemented")++ @abstractmethod+ def get_graph_entities(self) -> list[str]:+ """+ Get the entities of the graph.+ """+ raise NotImplementedError("get_graph_entities method not implemented")diff --git a/src/config.py b/src/config.pyindex 4c86514..925c8aa 100644--- a/src/config.py+++ b/src/config.py@@ -93,6 +93,21 @@ class EmbeddingsConfig:self.key = os.getenv("EMBEDDINGS_KEY")+class MongoConfig:+ """+ Configuration class for the Mongo configuration.+ """++ def __init__(self):+ self.host = os.getenv("MONGO_HOST")+ self.port = int(os.getenv("MONGO_PORT"))+ self.username = os.getenv("MONGO_USERNAME")+ self.password = os.getenv("MONGO_PASSWORD")++ if [self.host, self.port, self.username, self.password].count(None) > 0:+ raise ValueError("Mongo configuration is not complete")++class Config:"""Configuration class for the application.@@ -104,6 +119,7 @@ class Config:self.neo4j = Neo4jConfig()self.milvus = MilvusConfig()self.embeddings = EmbeddingsConfig()+ self.mongo = MongoConfig()config = Config()diff --git a/src/constants/data.py b/src/constants/data.pynew file mode 100644index 0000000..9066f8c--- /dev/null+++ b/src/constants/data.py@@ -0,0 +1,45 @@+"""+File: /data.py+Created Date: Saturday October 25th 2025+Author: Christian Nonis <[email protected]>+-----+Last Modified: Saturday October 25th 2025 11:48:51 am+Modified By: the developer formerly known as Christian Nonis at <[email protected]>+-----+"""++from datetime import datetime+from typing import Optional+import uuid+from pydantic import BaseModel, Field+++class TextChunk(BaseModel):+ """+ Text chunk model.+ """++ id: str = Field(default_factory=lambda: str(uuid.uuid4()))+ text: str = Field(description="The text of the chunk.")+ metadata: Optional[dict] = None+ inserted_at: datetime = Field(+ default_factory=datetime.now,+ description="The date and time the chunk was inserted.",+ )+++class Observation(BaseModel):+ """+ Observation model.+ """++ id: str = Field(default_factory=lambda: str(uuid.uuid4()))+ text: str = Field(description="The text of the observation.")+ metadata: Optional[dict] = None+ resource_id: str = Field(+ description="The id of the resource the observation is about."+ )+ inserted_at: datetime = Field(+ default_factory=datetime.now,+ description="The date and time the observation was inserted.",+ )diff --git a/src/constants/embeddings.py b/src/constants/embeddings.pyindex e4e5e68..ab3d88e 100644--- a/src/constants/embeddings.py+++ b/src/constants/embeddings.py@@ -30,7 +30,7 @@ class Vector(BaseModel):"""id: str- embeddings: Optional[list[float]]+ embeddings: Optional[list[float]] = Nonemetadata: dictdistance: Optional[float] = Field(diff --git a/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py b/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.pyindex 4a8525b..9b438fc 100644--- a/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py+++ b/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py@@ -80,9 +80,9 @@ class KGAgentAddTripletsTool(BaseTool):for triplet in triplets:vector = self.embeddings.embed_text(triplet.predicate.description)vector.metadata = {+ **(self.metadata or {}),"node_ids": [triplet.subject.uuid, triplet.object.uuid],"predicate": triplet.predicate.name,- **(self.metadata or {}),}self.vector_store.add_vectors([vector], "triplets")self.kg.add_nodes(diff --git a/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py b/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.pyindex ce28f56..391427a 100644--- a/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py+++ b/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py@@ -114,7 +114,9 @@ class KGAgentSearchGraphTool(BaseTool):with_relationships=True,relationships_depth=1,relationships_type=[- v_result.metadata["predicate"] for v_result in v_results+ v_result.metadata.get("predicate")+ for v_result in v_results+ if v_result.metadata.get("predicate", None)],))diff --git a/src/lib/milvus/client.py b/src/lib/milvus/client.pyindex 7ca8fe8..c7b12ed 100644--- a/src/lib/milvus/client.py+++ b/src/lib/milvus/client.py@@ -58,6 +58,7 @@ class MilvusClient(VectorStoreClient):Add vectors to the vector store."""self._ensure_store(store)+ print("adding vectors", vectors)self.client.insert(store,[@@ -78,19 +79,17 @@ class MilvusClient(VectorStoreClient):"""self._ensure_store(store)_results = self.client.search(- store,- data=[data_vector],- limit=k,+ store, data=[data_vector], limit=k, output_fields=["$meta"])results = []for query_results in _results:for result in query_results:results.append(Vector(- id=result["id"],+ id=str(result["id"]),metadata={k: v- for k, v in result.items()+ for k, v in result.get("entity").items()if k not in ["id", "embeddings", "distance"]},distance=result["distance"],diff --git a/src/lib/mongo/client.py b/src/lib/mongo/client.pynew file mode 100644index 0000000..1e5291e--- /dev/null+++ b/src/lib/mongo/client.py@@ -0,0 +1,117 @@+"""+File: /client.py+Created Date: Saturday October 25th 2025+Author: Christian Nonis <[email protected]>+-----+Last Modified: Saturday October 25th 2025 11:55:47 am+Modified By: the developer formerly known as Christian Nonis at <[email protected]>+-----+"""++from typing import List+from pymongo import MongoClient as PyMongoClient+from pymongo.database import Database+from pymongo.collection import Collection+from src.adapters.interfaces.data import DataClient, SearchResult+from src.config import config+from src.constants.data import Observation, TextChunk+++class MongoClient(DataClient):+ """+ Mongo client.+ """++ def __init__(self):+ self.client = PyMongoClient(+ config.mongo.host,+ config.mongo.port,+ username=config.mongo.username,+ password=config.mongo.password,+ authSource="admin",+ )++ def get_database(self, database: str = "data") -> Database:+ return self.client[database]++ def get_collection(self, collection: str, database: str = "data") -> Collection:+ return self.get_database(database)[collection]++ def save_text_chunk(self, text_chunk: TextChunk) -> TextChunk:+ collection = self.get_collection("text_chunks")+ collection.insert_one(text_chunk.model_dump(mode="json"))+ return text_chunk++ def save_observations(self, observations: List[Observation]) -> Observation:+ collection = self.get_collection("observations")+ collection.insert_many([o.model_dump(mode="json") for o in observations])+ return observations++ def search(self, text: str, collection: str = "*", limit: int = 10) -> SearchResult:+ chunks_collection = self.get_collection(collection)+ observations_collection = self.get_collection("observations")++ text_chunks_results = chunks_collection.find(+ {"text": {"$regex": text, "$options": "i"}}+ )+ text_chunks = [+ TextChunk(+ id=result["_id"],+ text=result["text"],+ metadata=result.get("metadata", None),+ )+ for result in text_chunks_results+ ]+ observations = observations_collection.find(+ {"resource_id": {"$in": [text_chunk.id for text_chunk in text_chunks]}}+ )+ observations = [+ Observation(+ id=result["id"],+ text=result["text"],+ metadata=result.get("metadata", None),+ resource_id=result["resource_id"],+ )+ for result in observations+ ]++ return SearchResult(text_chunks=text_chunks, observations=observations)++ def get_text_chunks_by_ids(+ self, ids: List[str], with_observations: bool = False+ ) -> List[TextChunk]:+ chunks_collection = self.get_collection("text_chunks")+ text_chunks = chunks_collection.find({"id": {"$in": ids}})++ if with_observations:+ observations_collection = self.get_collection("observations")+ observations = observations_collection.find({"resource_id": {"$in": ids}})++ return (+ [+ TextChunk(+ id=result["id"],+ text=result["text"],+ metadata=result.get("metadata", None),+ inserted_at=result["inserted_at"],+ )+ for result in text_chunks+ ],+ (+ [+ Observation(+ id=result["id"],+ text=result["text"],+ metadata=result.get("metadata", None),+ resource_id=result["resource_id"],+ inserted_at=result["inserted_at"],+ )+ for result in observations+ ]+ if with_observations+ else []+ ),+ )+++_mongo_client = MongoClient()diff --git a/src/lib/mongo/docker-compose.yaml b/src/lib/mongo/docker-compose.yamlnew file mode 100644index 0000000..1264e5f--- /dev/null+++ b/src/lib/mongo/docker-compose.yaml@@ -0,0 +1,34 @@+services:+ mongo:+ image: mongo:8.0+ container_name: mongodb+ restart: always+ ports:+ - "27017:27017"+ environment:+ MONGO_INITDB_ROOT_USERNAME: root+ MONGO_INITDB_ROOT_PASSWORD: password+ volumes:+ - mongo-data:/data/db+ - mongo-config:/data/configdb++ mongo-express:+ image: mongo-express:latest+ container_name: mongo-express+ restart: always+ ports:+ - "8081:8081"+ environment:+ ME_CONFIG_MONGODB_ADMINUSERNAME: root+ ME_CONFIG_MONGODB_ADMINPASSWORD: password+ ME_CONFIG_MONGODB_URL: mongodb://root:password@mongo:27017/+ ME_CONFIG_BASICAUTH_USERNAME: admin+ ME_CONFIG_BASICAUTH_PASSWORD: admin+ depends_on:+ - mongo++volumes:+ mongo-data:+ driver: local+ mongo-config:+ driver: local\ No newline at end of filediff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.pyindex 4a02b4b..ad534a7 100644--- a/src/lib/neo4j/client.py+++ b/src/lib/neo4j/client.py@@ -151,7 +151,7 @@ class Neo4jClient(GraphClient):Node(uuid=node["uuid"],name=node["name"],- label=node["label"],+ label=(node["labels"][0] if node["labels"] else None),description=node["description"],properties=node["properties"],)@@ -164,6 +164,7 @@ class Neo4jClient(GraphClient):with_relationships: Optional[bool] = False,relationships_depth: Optional[int] = 1,relationships_type: Optional[list[str]] = None,+ preferred_labels: Optional[list[str]] = None,) -> list[Node]:"""Get nodes by their UUIDs with optional relationships.@@ -182,6 +183,14 @@ class Neo4jClient(GraphClient):cypher_query += f"""OPTIONAL MATCH (n)-[r*1..{relationships_depth}]-(m)"""+ if preferred_labels and len(preferred_labels) > 0:+ cypher_query += f"""+ WHERE any(lbl IN labels(m) WHERE lbl IN {preferred_labels})+ """+ cypher_query += """+ WITH n, r, m+ WHERE r IS NOT NULL OR m IS NOT NULL+ """cypher_query += """RETURN n.uuid as uuid, n.name as name, labels(n) as labels, n.description as description, properties(n) as properties@@ -219,5 +228,16 @@ class Neo4jClient(GraphClient):for record in result.records]+ def get_graph_entities(self) -> list[str]:+ """+ Get the entities of the graph.+ """+ cypher_query = """+ MATCH (n)+ RETURN DISTINCT labels(n) as labels+ """+ result = self.driver.execute_query(cypher_query)+ return [label for record in result.records for label in record["labels"]]+_neo4j_client = Neo4jClient()diff --git a/src/services/api/app.py b/src/services/api/app.pyindex 87c3564..a13030d 100644--- a/src/services/api/app.py+++ b/src/services/api/app.py@@ -14,10 +14,12 @@ from fastapi import FastAPIfrom uvicorn import runfrom src.services.api.routes.ingest import ingest_router+from src.services.api.routes.retrieve import retrieve_routerapp = FastAPI()app.include_router(ingest_router)+app.include_router(retrieve_router)if __name__ == "__main__":run(app, host="0.0.0.0", port=8000, reload=os.getenv("ENV") == "development")diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.pyindex 0f06f32..4c05a76 100644--- a/src/services/api/constants/requests.py+++ b/src/services/api/constants/requests.py@@ -8,6 +8,9 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@-----"""+from typing import List, Any+from pydantic import BaseModel, field_serializer+from src.constants.data import Observation, TextChunkfrom src.constants.tasks.ingestion import IngestionTaskArgs@@ -15,3 +18,64 @@ class IngestionRequestBody(IngestionTaskArgs):"""Request body for the ingestion endpoint."""+++class RetrieveRequestResponse(BaseModel):+ """+ Response for the retrieve endpoint.+ """++ data: List[TextChunk]+ observations: List[Observation]+ relationships: List[dict]++ @field_serializer("relationships", when_used="json")+ def _serialize_relationships(self, value: List[Any]):+ try:+ from neo4j.graph import (+ Node as NeoNode,+ Relationship as NeoRel,+ Path as NeoPath,+ )+ except Exception:+ NeoNode = NeoRel = NeoPath = tuple()+ from src.constants.kg import Node as KGNode++ def _ser(obj):+ if obj is None:+ return None+ if isinstance(obj, (str, int, float, bool)):+ return obj+ if isinstance(obj, KGNode):+ return obj.model_dump(mode="json")+ if isinstance(obj, dict):+ return {k: _ser(v) for k, v in obj.items()}+ if isinstance(obj, (list, tuple, set)):+ return [_ser(v) for v in obj]+ if NeoNode and isinstance(obj, NeoNode):+ d = dict(obj)+ labels = getattr(obj, "labels", None)+ if labels is not None:+ d["_labels"] = list(labels)+ return d+ if NeoRel and isinstance(obj, NeoRel):+ d = dict(obj)+ rel_type = getattr(obj, "type", None)+ if rel_type is not None:+ d["_type"] = rel_type+ try:+ nodes_attr = getattr(obj, "nodes", None)+ if nodes_attr:+ d["_start_id"] = getattr(nodes_attr[0], "id", None)+ d["_end_id"] = getattr(nodes_attr[1], "id", None)+ except Exception:+ pass+ return d+ if NeoPath and isinstance(obj, NeoPath):+ return {+ "nodes": [_ser(n) for n in obj.nodes],+ "relationships": [_ser(r) for r in obj.relationships],+ }+ return str(obj)++ return [_ser(v) for v in value] if value is not None else []diff --git a/src/services/api/routes/retrieve.py b/src/services/api/routes/retrieve.pynew file mode 100644index 0000000..ac6d493--- /dev/null+++ b/src/services/api/routes/retrieve.py@@ -0,0 +1,81 @@+"""+File: /retrieve.py+Created Date: Saturday October 25th 2025+Author: Christian Nonis <[email protected]>+-----+Last Modified: Saturday October 25th 2025 12:32:21 pm+Modified By: the developer formerly known as Christian Nonis at <[email protected]>+-----+"""++import asyncio+from typing import List+from fastapi import APIRouter, Query++from src.services.api.constants.requests import RetrieveRequestResponse+from src.services.data.main import data_adapter+from src.services.kg_agent.main import embeddings_adapter, vector_store_adapter+from src.services.kg_agent.main import graph_adapter++retrieve_router = APIRouter(prefix="/retrieve", tags=["retrieve"])+++@retrieve_router.get("/", response_model=RetrieveRequestResponse)+async def retrieve(+ text: str = Query(..., description="The text to search for."),+ limit: int = Query(10, description="The number of results to return."),+ preferred_entities: List[str] = Query(+ ..., description="The entities to prioritize in the relationships."+ ),+):+ """+ Retrieve data from the knowledge graph and data store.+ """++ def _get_data():+ text_embeddings = embeddings_adapter.embed_text(text)+ data_vectors = vector_store_adapter.search_vectors(+ text_embeddings.embeddings, "data", limit+ )+ triple_vectors = vector_store_adapter.search_vectors(+ text_embeddings.embeddings, "triplets", limit+ )++ search_result = data_adapter.search(text)++ ts_text_chunks = search_result.text_chunks+ ts_observations = search_result.observations++ v_text_chunks, v_observations = data_adapter.get_text_chunks_by_ids(+ [dv.metadata.get("resource_id") for dv in data_vectors], True+ )++ node_ids = [+ node_id+ for tv in triple_vectors+ for node_id in tv.metadata.get("node_ids", [])+ ]++ nodes = graph_adapter.get_nodes_by_uuid(+ uuids=node_ids,+ with_relationships=True,+ relationships_depth=1,+ relationships_type=[+ tv.metadata.get("predicate")+ for tv in triple_vectors+ if tv.metadata.get("predicate", None)+ ],+ preferred_labels=preferred_entities or [],+ )++ return ts_text_chunks, ts_observations, v_text_chunks, v_observations, nodes++ ts_text_chunks, ts_observations, v_text_chunks, v_observations, nodes = (+ await asyncio.to_thread(_get_data)+ )++ return RetrieveRequestResponse(+ data=[*ts_text_chunks, *v_text_chunks],+ observations=[*ts_observations, *v_observations],+ relationships=nodes,+ )diff --git a/src/services/data/__init__.py b/src/services/data/__init__.pynew file mode 100644index 0000000..560f613--- /dev/null+++ b/src/services/data/__init__.py@@ -0,0 +1,9 @@+"""+File: /__init__.py+Created Date: Saturday October 25th 2025+Author: Christian Nonis <[email protected]>+-----+Last Modified: Saturday October 25th 2025 12:11:04 pm+Modified By: the developer formerly known as Christian Nonis at <[email protected]>+-----+"""diff --git a/src/services/data/main.py b/src/services/data/main.pynew file mode 100644index 0000000..e25aef1--- /dev/null+++ b/src/services/data/main.py@@ -0,0 +1,15 @@+"""+File: /main.py+Created Date: Saturday October 25th 2025+Author: Christian Nonis <[email protected]>+-----+Last Modified: Saturday October 25th 2025 12:11:27 pm+Modified By: the developer formerly known as Christian Nonis at <[email protected]>+-----+"""++from src.adapters.data import DataAdapter+from src.lib.mongo.client import _mongo_client++data_adapter = DataAdapter()+data_adapter.add_client(_mongo_client)diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.pyindex b1a7e46..fb07a96 100644--- a/src/workers/tasks/ingestion.py+++ b/src/workers/tasks/ingestion.py@@ -9,7 +9,13 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@"""import json-from src.services.kg_agent.main import kg_agent+from src.constants.data import Observation, TextChunk+from src.services.data.main import data_adapter+from src.services.kg_agent.main import (+ embeddings_adapter,+ kg_agent,+ vector_store_adapter,+)from src.services.observations.main import observations_agentfrom src.workers.app import ingestion_appfrom src.constants.tasks.ingestion import (@@ -28,7 +34,25 @@ def ingest_data(self, args: dict):# ================================================# --------------- Data Saving --------------------# ================================================- pass+ text_chunk = data_adapter.save_text_chunk(+ TextChunk(+ text=(+ payload.data.text_data+ if payload.data.data_type == IngestionTaskDataType.TEXT.value+ else json.dumps(payload.data.json_data)+ ),+ metadata=payload.meta_keys,+ )+ )+ text_chunk_vector = embeddings_adapter.embed_text(text_chunk.text)+ text_chunk_vector.metadata = {+ **payload.meta_keys,+ "resource_id": text_chunk.id,+ }+ vector_store_adapter.add_vectors(+ [text_chunk_vector],+ "data",+ )# ================================================# --------------- Observations -------------------@@ -41,6 +65,16 @@ def ingest_data(self, args: dict):),observate_for=payload.observate_for,)+ data_adapter.save_observations(+ [+ Observation(+ text=observation,+ metadata=payload.meta_keys,+ resource_id=text_chunk.id,+ )+ for observation in observations+ ]+ )# ================================================# ------------ Triplet Extraction ----------------@@ -56,7 +90,7 @@ def ingest_data(self, args: dict):kg_agent.update_kg(information=information,- metadata=payload.meta_keys,+ metadata={**payload.meta_keys, "resource_id": text_chunk.id},identification_params=payload.identification_params,)