Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

feat(read/write)

Christian422e520ecd6e
diff --git a/Makefile b/Makefile
index a164a18..6720a61 100644
--- a/Makefile
+++ b/Makefile
@@ -35,6 +35,15 @@ stop-neo4j:
delete-neo4j-volumes:
docker compose -f src/lib/neo4j/docker-compose.yaml down -v --remove-orphans
+start-mongo:
+ docker compose -f src/lib/mongo/docker-compose.yaml up -d
+
+stop-mongo:
+ docker compose -f src/lib/mongo/docker-compose.yaml down
+
+delete-mongo-volumes:
+ docker compose -f src/lib/mongo/docker-compose.yaml down -v --remove-orphans
+
start-api:
poetry run uvicorn src.services.api.app:app --host 0.0.0.0 --port 8000
@@ -52,6 +61,7 @@ start-all:
make start-rabbitmq &
make start-redis &
make start-neo4j &
+ make start-mongo &
make start-api &
@bash -c "poetry run celery -A src.workers.app worker --loglevel=info"
@@ -60,6 +70,7 @@ stop-all:
make stop-rabbitmq
make stop-redis
make stop-neo4j
+ make stop-mongo
make stop-api
pkill -f celery
@@ -67,4 +78,5 @@ clear-all-volumes:
make delete-milvus-volumes
make delete-rabbitmq-volumes
make delete-redis-volumes
- make delete-neo4j-volumes
\ No newline at end of file
+ make delete-neo4j-volumes
+ make delete-mongo-volumes
\ No newline at end of file
diff --git a/src/adapters/data.py b/src/adapters/data.py
index 708d5db..1acc111 100644
--- a/src/adapters/data.py
+++ b/src/adapters/data.py
@@ -7,3 +7,48 @@ Last Modified: Sunday October 19th 2025 9:00:42 am
Modified By: the developer formerly known as Christian Nonis at <[email protected]>
-----
"""
+
+from typing import List, Tuple
+from src.adapters.interfaces.data import DataClient, SearchResult
+from src.constants.data import Observation, TextChunk
+
+
+class DataAdapter:
+ """
+ Data adapter.
+ """
+
+ def __init__(self):
+ self.data = None
+
+ def add_client(self, client: DataClient) -> None:
+ """
+ Add a data client to the adapter.
+ """
+ self.data = client
+
+ def save_text_chunk(self, text_chunk: TextChunk) -> TextChunk:
+ """
+ Save a text chunk to the data client.
+ """
+ return self.data.save_text_chunk(text_chunk)
+
+ def save_observations(self, observations: List[Observation]) -> Observation:
+ """
+ Save a list of observations to the data client.
+ """
+ return self.data.save_observations(observations)
+
+ def search(self, text: str) -> SearchResult:
+ """
+ Search data by text and return a list of text chunks and observations.
+ """
+ return self.data.search(text)
+
+ def get_text_chunks_by_ids(
+ self, ids: List[str], with_observations: bool
+ ) -> Tuple[List[TextChunk], List[Observation]]:
+ """
+ Get data by their IDs.
+ """
+ return self.data.get_text_chunks_by_ids(ids, with_observations)
diff --git a/src/adapters/graph.py b/src/adapters/graph.py
index 7afb768..ed16ae6 100644
--- a/src/adapters/graph.py
+++ b/src/adapters/graph.py
@@ -96,13 +96,24 @@ class GraphAdapter:
with_relationships: Optional[bool] = False,
relationships_depth: Optional[int] = 1,
relationships_type: Optional[list[str]] = None,
+ preferred_labels: Optional[list[str]] = None,
) -> list[Node]:
"""
- Get nodes by their UUIDs.
+ Get nodes by their UUIDs with optional relationships and preferred labels.
"""
return self.graph.get_nodes_by_uuid(
- uuids, with_relationships, relationships_depth, relationships_type
+ uuids,
+ with_relationships,
+ relationships_depth,
+ relationships_type,
+ preferred_labels,
)
+ def get_graph_entities(self) -> list[str]:
+ """
+ Get the entities of the graph.
+ """
+ return self.graph.get_graph_entities()
+
_graph_adapter = GraphAdapter()
diff --git a/src/adapters/interfaces/data.py b/src/adapters/interfaces/data.py
index 7492275..6f426f0 100644
--- a/src/adapters/interfaces/data.py
+++ b/src/adapters/interfaces/data.py
@@ -7,3 +7,55 @@ Last Modified: Sunday October 19th 2025 9:01:13 am
Modified By: the developer formerly known as Christian Nonis at <[email protected]>
-----
"""
+
+from abc import ABC, abstractmethod
+from typing import List, Tuple
+
+from pydantic import BaseModel
+
+from src.constants.data import Observation, TextChunk
+
+
+class SearchResult(BaseModel):
+ """
+ Search result model.
+ """
+
+ text_chunks: List[TextChunk]
+ observations: List[Observation]
+
+
+class DataClient(ABC):
+ """
+ Abstract base class for data clients.
+ """
+
+ @abstractmethod
+ def save_text_chunk(self, text_chunk: TextChunk) -> TextChunk:
+ """
+ Save data to the data client.
+ """
+ raise NotImplementedError("save method not implemented")
+
+ @abstractmethod
+ def save_observations(self, observations: List[Observation]) -> Observation:
+ """
+ Save a list of observations to the data client.
+ """
+ raise NotImplementedError("save_observations method not implemented")
+
+ @abstractmethod
+ def search(self, text: str) -> SearchResult:
+ """
+ Search data by text and return a list of text chunks and observations.
+ """
+ raise NotImplementedError("search method not implemented")
+
+ @abstractmethod
+ def get_text_chunks_by_ids(
+ self, ids: List[str], with_observations: bool
+ ) -> Tuple[List[TextChunk], List[Observation]]:
+ """
+ Get data by their IDs.
+ """
+ raise NotImplementedError("get_by_ids method not implemented")
diff --git a/src/adapters/interfaces/graph.py b/src/adapters/interfaces/graph.py
index 3fa64cd..3ae7e6e 100644
--- a/src/adapters/interfaces/graph.py
+++ b/src/adapters/interfaces/graph.py
@@ -90,8 +90,16 @@ class GraphClient(ABC):
with_relationships: Optional[bool] = False,
relationships_depth: Optional[int] = 1,
relationships_type: Optional[list[str]] = None,
+ preferred_labels: Optional[list[str]] = None,
) -> list[Node]:
"""
- Get nodes by their UUIDs with optional relationships.
+ Get nodes by their UUIDs with optional relationships and preferred labels.
"""
raise NotImplementedError("get_nodes_by_uuid method not implemented")
+
+ @abstractmethod
+ def get_graph_entities(self) -> list[str]:
+ """
+ Get the entities of the graph.
+ """
+ raise NotImplementedError("get_graph_entities method not implemented")
diff --git a/src/config.py b/src/config.py
index 4c86514..925c8aa 100644
--- a/src/config.py
+++ b/src/config.py
@@ -93,6 +93,21 @@ class EmbeddingsConfig:
self.key = os.getenv("EMBEDDINGS_KEY")
+class MongoConfig:
+ """
+ Configuration class for the Mongo configuration.
+ """
+
+ def __init__(self):
+ self.host = os.getenv("MONGO_HOST")
+ self.port = int(os.getenv("MONGO_PORT"))
+ self.username = os.getenv("MONGO_USERNAME")
+ self.password = os.getenv("MONGO_PASSWORD")
+
+ if [self.host, self.port, self.username, self.password].count(None) > 0:
+ raise ValueError("Mongo configuration is not complete")
+
+
class Config:
"""
Configuration class for the application.
@@ -104,6 +119,7 @@ class Config:
self.neo4j = Neo4jConfig()
self.milvus = MilvusConfig()
self.embeddings = EmbeddingsConfig()
+ self.mongo = MongoConfig()
config = Config()
diff --git a/src/constants/data.py b/src/constants/data.py
new file mode 100644
index 0000000..9066f8c
--- /dev/null
+++ b/src/constants/data.py
@@ -0,0 +1,45 @@
+"""
+File: /data.py
+Created Date: Saturday October 25th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Saturday October 25th 2025 11:48:51 am
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
+
+from datetime import datetime
+from typing import Optional
+import uuid
+from pydantic import BaseModel, Field
+
+
+class TextChunk(BaseModel):
+ """
+ Text chunk model.
+ """
+
+ id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+ text: str = Field(description="The text of the chunk.")
+ metadata: Optional[dict] = None
+ inserted_at: datetime = Field(
+ default_factory=datetime.now,
+ description="The date and time the chunk was inserted.",
+ )
+
+
+class Observation(BaseModel):
+ """
+ Observation model.
+ """
+
+ id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+ text: str = Field(description="The text of the observation.")
+ metadata: Optional[dict] = None
+ resource_id: str = Field(
+ description="The id of the resource the observation is about."
+ )
+ inserted_at: datetime = Field(
+ default_factory=datetime.now,
+ description="The date and time the observation was inserted.",
+ )
diff --git a/src/constants/embeddings.py b/src/constants/embeddings.py
index e4e5e68..ab3d88e 100644
--- a/src/constants/embeddings.py
+++ b/src/constants/embeddings.py
@@ -30,7 +30,7 @@ class Vector(BaseModel):
"""
id: str
- embeddings: Optional[list[float]]
+ embeddings: Optional[list[float]] = None
metadata: dict
distance: Optional[float] = Field(
diff --git a/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py b/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py
index 4a8525b..9b438fc 100644
--- a/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py
+++ b/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py
@@ -80,9 +80,9 @@ class KGAgentAddTripletsTool(BaseTool):
for triplet in triplets:
vector = self.embeddings.embed_text(triplet.predicate.description)
vector.metadata = {
+ **(self.metadata or {}),
"node_ids": [triplet.subject.uuid, triplet.object.uuid],
"predicate": triplet.predicate.name,
- **(self.metadata or {}),
}
self.vector_store.add_vectors([vector], "triplets")
self.kg.add_nodes(
diff --git a/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py b/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py
index ce28f56..391427a 100644
--- a/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py
+++ b/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py
@@ -114,7 +114,9 @@ class KGAgentSearchGraphTool(BaseTool):
with_relationships=True,
relationships_depth=1,
relationships_type=[
- v_result.metadata["predicate"] for v_result in v_results
+ v_result.metadata.get("predicate")
+ for v_result in v_results
+ if v_result.metadata.get("predicate", None)
],
)
)
diff --git a/src/lib/milvus/client.py b/src/lib/milvus/client.py
index 7ca8fe8..c7b12ed 100644
--- a/src/lib/milvus/client.py
+++ b/src/lib/milvus/client.py
@@ -58,6 +58,7 @@ class MilvusClient(VectorStoreClient):
Add vectors to the vector store.
"""
self._ensure_store(store)
+ print("adding vectors", vectors)
self.client.insert(
store,
[
@@ -78,19 +79,17 @@ class MilvusClient(VectorStoreClient):
"""
self._ensure_store(store)
_results = self.client.search(
- store,
- data=[data_vector],
- limit=k,
+ store, data=[data_vector], limit=k, output_fields=["$meta"]
)
results = []
for query_results in _results:
for result in query_results:
results.append(
Vector(
- id=result["id"],
+ id=str(result["id"]),
metadata={
k: v
- for k, v in result.items()
+ for k, v in result.get("entity").items()
if k not in ["id", "embeddings", "distance"]
},
distance=result["distance"],
diff --git a/src/lib/mongo/client.py b/src/lib/mongo/client.py
new file mode 100644
index 0000000..1e5291e
--- /dev/null
+++ b/src/lib/mongo/client.py
@@ -0,0 +1,117 @@
+"""
+File: /client.py
+Created Date: Saturday October 25th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Saturday October 25th 2025 11:55:47 am
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
+
+from typing import List
+from pymongo import MongoClient as PyMongoClient
+from pymongo.database import Database
+from pymongo.collection import Collection
+from src.adapters.interfaces.data import DataClient, SearchResult
+from src.config import config
+from src.constants.data import Observation, TextChunk
+
+
+class MongoClient(DataClient):
+ """
+ Mongo client.
+ """
+
+ def __init__(self):
+ self.client = PyMongoClient(
+ config.mongo.host,
+ config.mongo.port,
+ username=config.mongo.username,
+ password=config.mongo.password,
+ authSource="admin",
+ )
+
+ def get_database(self, database: str = "data") -> Database:
+ return self.client[database]
+
+ def get_collection(self, collection: str, database: str = "data") -> Collection:
+ return self.get_database(database)[collection]
+
+ def save_text_chunk(self, text_chunk: TextChunk) -> TextChunk:
+ collection = self.get_collection("text_chunks")
+ collection.insert_one(text_chunk.model_dump(mode="json"))
+ return text_chunk
+
+ def save_observations(self, observations: List[Observation]) -> Observation:
+ collection = self.get_collection("observations")
+ collection.insert_many([o.model_dump(mode="json") for o in observations])
+ return observations
+
+ def search(self, text: str, collection: str = "*", limit: int = 10) -> SearchResult:
+ chunks_collection = self.get_collection(collection)
+ observations_collection = self.get_collection("observations")
+
+ text_chunks_results = chunks_collection.find(
+ {"text": {"$regex": text, "$options": "i"}}
+ )
+ text_chunks = [
+ TextChunk(
+ id=result["_id"],
+ text=result["text"],
+ metadata=result.get("metadata", None),
+ )
+ for result in text_chunks_results
+ ]
+ observations = observations_collection.find(
+ {"resource_id": {"$in": [text_chunk.id for text_chunk in text_chunks]}}
+ )
+ observations = [
+ Observation(
+ id=result["id"],
+ text=result["text"],
+ metadata=result.get("metadata", None),
+ resource_id=result["resource_id"],
+ )
+ for result in observations
+ ]
+
+ return SearchResult(text_chunks=text_chunks, observations=observations)
+
+ def get_text_chunks_by_ids(
+ self, ids: List[str], with_observations: bool = False
+ ) -> List[TextChunk]:
+ chunks_collection = self.get_collection("text_chunks")
+ text_chunks = chunks_collection.find({"id": {"$in": ids}})
+
+ if with_observations:
+ observations_collection = self.get_collection("observations")
+ observations = observations_collection.find({"resource_id": {"$in": ids}})
+
+ return (
+ [
+ TextChunk(
+ id=result["id"],
+ text=result["text"],
+ metadata=result.get("metadata", None),
+ inserted_at=result["inserted_at"],
+ )
+ for result in text_chunks
+ ],
+ (
+ [
+ Observation(
+ id=result["id"],
+ text=result["text"],
+ metadata=result.get("metadata", None),
+ resource_id=result["resource_id"],
+ inserted_at=result["inserted_at"],
+ )
+ for result in observations
+ ]
+ if with_observations
+ else []
+ ),
+ )
+
+
+_mongo_client = MongoClient()
diff --git a/src/lib/mongo/docker-compose.yaml b/src/lib/mongo/docker-compose.yaml
new file mode 100644
index 0000000..1264e5f
--- /dev/null
+++ b/src/lib/mongo/docker-compose.yaml
@@ -0,0 +1,34 @@
+services:
+ mongo:
+ image: mongo:8.0
+ container_name: mongodb
+ restart: always
+ ports:
+ - "27017:27017"
+ environment:
+ MONGO_INITDB_ROOT_USERNAME: root
+ MONGO_INITDB_ROOT_PASSWORD: password
+ volumes:
+ - mongo-data:/data/db
+ - mongo-config:/data/configdb
+
+ mongo-express:
+ image: mongo-express:latest
+ container_name: mongo-express
+ restart: always
+ ports:
+ - "8081:8081"
+ environment:
+ ME_CONFIG_MONGODB_ADMINUSERNAME: root
+ ME_CONFIG_MONGODB_ADMINPASSWORD: password
+ ME_CONFIG_MONGODB_URL: mongodb://root:password@mongo:27017/
+ ME_CONFIG_BASICAUTH_USERNAME: admin
+ ME_CONFIG_BASICAUTH_PASSWORD: admin
+ depends_on:
+ - mongo
+
+volumes:
+ mongo-data:
+ driver: local
+ mongo-config:
+ driver: local
\ No newline at end of file
diff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.py
index 4a02b4b..ad534a7 100644
--- a/src/lib/neo4j/client.py
+++ b/src/lib/neo4j/client.py
@@ -151,7 +151,7 @@ class Neo4jClient(GraphClient):
Node(
uuid=node["uuid"],
name=node["name"],
- label=node["label"],
+ label=(node["labels"][0] if node["labels"] else None),
description=node["description"],
properties=node["properties"],
)
@@ -164,6 +164,7 @@ class Neo4jClient(GraphClient):
with_relationships: Optional[bool] = False,
relationships_depth: Optional[int] = 1,
relationships_type: Optional[list[str]] = None,
+ preferred_labels: Optional[list[str]] = None,
) -> list[Node]:
"""
Get nodes by their UUIDs with optional relationships.
@@ -182,6 +183,14 @@ class Neo4jClient(GraphClient):
cypher_query += f"""
OPTIONAL MATCH (n)-[r*1..{relationships_depth}]-(m)
"""
+ if preferred_labels and len(preferred_labels) > 0:
+ cypher_query += f"""
+ WHERE any(lbl IN labels(m) WHERE lbl IN {preferred_labels})
+ """
+ cypher_query += """
+ WITH n, r, m
+ WHERE r IS NOT NULL OR m IS NOT NULL
+ """
cypher_query += """
RETURN n.uuid as uuid, n.name as name, labels(n) as labels, n.description as description, properties(n) as properties
@@ -219,5 +228,16 @@ class Neo4jClient(GraphClient):
for record in result.records
]
+ def get_graph_entities(self) -> list[str]:
+ """
+ Get the entities of the graph.
+ """
+ cypher_query = """
+ MATCH (n)
+ RETURN DISTINCT labels(n) as labels
+ """
+ result = self.driver.execute_query(cypher_query)
+ return [label for record in result.records for label in record["labels"]]
+
_neo4j_client = Neo4jClient()
diff --git a/src/services/api/app.py b/src/services/api/app.py
index 87c3564..a13030d 100644
--- a/src/services/api/app.py
+++ b/src/services/api/app.py
@@ -14,10 +14,12 @@ from fastapi import FastAPI
from uvicorn import run
from src.services.api.routes.ingest import ingest_router
+from src.services.api.routes.retrieve import retrieve_router
app = FastAPI()
app.include_router(ingest_router)
+app.include_router(retrieve_router)
if __name__ == "__main__":
run(app, host="0.0.0.0", port=8000, reload=os.getenv("ENV") == "development")
diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.py
index 0f06f32..4c05a76 100644
--- a/src/services/api/constants/requests.py
+++ b/src/services/api/constants/requests.py
@@ -8,6 +8,9 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
-----
"""
+from typing import List, Any
+from pydantic import BaseModel, field_serializer
+from src.constants.data import Observation, TextChunk
from src.constants.tasks.ingestion import IngestionTaskArgs
@@ -15,3 +18,64 @@ class IngestionRequestBody(IngestionTaskArgs):
"""
Request body for the ingestion endpoint.
"""
+
+
+class RetrieveRequestResponse(BaseModel):
+ """
+ Response for the retrieve endpoint.
+ """
+
+ data: List[TextChunk]
+ observations: List[Observation]
+ relationships: List[dict]
+
+ @field_serializer("relationships", when_used="json")
+ def _serialize_relationships(self, value: List[Any]):
+ try:
+ from neo4j.graph import (
+ Node as NeoNode,
+ Relationship as NeoRel,
+ Path as NeoPath,
+ )
+ except Exception:
+ NeoNode = NeoRel = NeoPath = tuple()
+ from src.constants.kg import Node as KGNode
+
+ def _ser(obj):
+ if obj is None:
+ return None
+ if isinstance(obj, (str, int, float, bool)):
+ return obj
+ if isinstance(obj, KGNode):
+ return obj.model_dump(mode="json")
+ if isinstance(obj, dict):
+ return {k: _ser(v) for k, v in obj.items()}
+ if isinstance(obj, (list, tuple, set)):
+ return [_ser(v) for v in obj]
+ if NeoNode and isinstance(obj, NeoNode):
+ d = dict(obj)
+ labels = getattr(obj, "labels", None)
+ if labels is not None:
+ d["_labels"] = list(labels)
+ return d
+ if NeoRel and isinstance(obj, NeoRel):
+ d = dict(obj)
+ rel_type = getattr(obj, "type", None)
+ if rel_type is not None:
+ d["_type"] = rel_type
+ try:
+ nodes_attr = getattr(obj, "nodes", None)
+ if nodes_attr:
+ d["_start_id"] = getattr(nodes_attr[0], "id", None)
+ d["_end_id"] = getattr(nodes_attr[1], "id", None)
+ except Exception:
+ pass
+ return d
+ if NeoPath and isinstance(obj, NeoPath):
+ return {
+ "nodes": [_ser(n) for n in obj.nodes],
+ "relationships": [_ser(r) for r in obj.relationships],
+ }
+ return str(obj)
+
+ return [_ser(v) for v in value] if value is not None else []
diff --git a/src/services/api/routes/retrieve.py b/src/services/api/routes/retrieve.py
new file mode 100644
index 0000000..ac6d493
--- /dev/null
+++ b/src/services/api/routes/retrieve.py
@@ -0,0 +1,81 @@
+"""
+File: /retrieve.py
+Created Date: Saturday October 25th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Saturday October 25th 2025 12:32:21 pm
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
+
+import asyncio
+from typing import List
+from fastapi import APIRouter, Query
+
+from src.services.api.constants.requests import RetrieveRequestResponse
+from src.services.data.main import data_adapter
+from src.services.kg_agent.main import embeddings_adapter, vector_store_adapter
+from src.services.kg_agent.main import graph_adapter
+
+retrieve_router = APIRouter(prefix="/retrieve", tags=["retrieve"])
+
+
+@retrieve_router.get("/", response_model=RetrieveRequestResponse)
+async def retrieve(
+ text: str = Query(..., description="The text to search for."),
+ limit: int = Query(10, description="The number of results to return."),
+ preferred_entities: List[str] = Query(
+ ..., description="The entities to prioritize in the relationships."
+ ),
+):
+ """
+ Retrieve data from the knowledge graph and data store.
+ """
+
+ def _get_data():
+ text_embeddings = embeddings_adapter.embed_text(text)
+ data_vectors = vector_store_adapter.search_vectors(
+ text_embeddings.embeddings, "data", limit
+ )
+ triple_vectors = vector_store_adapter.search_vectors(
+ text_embeddings.embeddings, "triplets", limit
+ )
+
+ search_result = data_adapter.search(text)
+
+ ts_text_chunks = search_result.text_chunks
+ ts_observations = search_result.observations
+
+ v_text_chunks, v_observations = data_adapter.get_text_chunks_by_ids(
+ [dv.metadata.get("resource_id") for dv in data_vectors], True
+ )
+
+ node_ids = [
+ node_id
+ for tv in triple_vectors
+ for node_id in tv.metadata.get("node_ids", [])
+ ]
+
+ nodes = graph_adapter.get_nodes_by_uuid(
+ uuids=node_ids,
+ with_relationships=True,
+ relationships_depth=1,
+ relationships_type=[
+ tv.metadata.get("predicate")
+ for tv in triple_vectors
+ if tv.metadata.get("predicate", None)
+ ],
+ preferred_labels=preferred_entities or [],
+ )
+
+ return ts_text_chunks, ts_observations, v_text_chunks, v_observations, nodes
+
+ ts_text_chunks, ts_observations, v_text_chunks, v_observations, nodes = (
+ await asyncio.to_thread(_get_data)
+ )
+
+ return RetrieveRequestResponse(
+ data=[*ts_text_chunks, *v_text_chunks],
+ observations=[*ts_observations, *v_observations],
+ relationships=nodes,
+ )
diff --git a/src/services/data/__init__.py b/src/services/data/__init__.py
new file mode 100644
index 0000000..560f613
--- /dev/null
+++ b/src/services/data/__init__.py
@@ -0,0 +1,9 @@
+"""
+File: /__init__.py
+Created Date: Saturday October 25th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Saturday October 25th 2025 12:11:04 pm
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
diff --git a/src/services/data/main.py b/src/services/data/main.py
new file mode 100644
index 0000000..e25aef1
--- /dev/null
+++ b/src/services/data/main.py
@@ -0,0 +1,15 @@
+"""
+File: /main.py
+Created Date: Saturday October 25th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Saturday October 25th 2025 12:11:27 pm
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
+
+from src.adapters.data import DataAdapter
+from src.lib.mongo.client import _mongo_client
+
+data_adapter = DataAdapter()
+data_adapter.add_client(_mongo_client)
diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.py
index b1a7e46..fb07a96 100644
--- a/src/workers/tasks/ingestion.py
+++ b/src/workers/tasks/ingestion.py
@@ -9,7 +9,13 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
"""
import json
-from src.services.kg_agent.main import kg_agent
+from src.constants.data import Observation, TextChunk
+from src.services.data.main import data_adapter
+from src.services.kg_agent.main import (
+ embeddings_adapter,
+ kg_agent,
+ vector_store_adapter,
+)
from src.services.observations.main import observations_agent
from src.workers.app import ingestion_app
from src.constants.tasks.ingestion import (
@@ -28,7 +34,25 @@ def ingest_data(self, args: dict):
# ================================================
# --------------- Data Saving --------------------
# ================================================
- pass
+ text_chunk = data_adapter.save_text_chunk(
+ TextChunk(
+ text=(
+ payload.data.text_data
+ if payload.data.data_type == IngestionTaskDataType.TEXT.value
+ else json.dumps(payload.data.json_data)
+ ),
+ metadata=payload.meta_keys,
+ )
+ )
+ text_chunk_vector = embeddings_adapter.embed_text(text_chunk.text)
+ text_chunk_vector.metadata = {
+ **payload.meta_keys,
+ "resource_id": text_chunk.id,
+ }
+ vector_store_adapter.add_vectors(
+ [text_chunk_vector],
+ "data",
+ )
# ================================================
# --------------- Observations -------------------
@@ -41,6 +65,16 @@ def ingest_data(self, args: dict):
),
observate_for=payload.observate_for,
)
+ data_adapter.save_observations(
+ [
+ Observation(
+ text=observation,
+ metadata=payload.meta_keys,
+ resource_id=text_chunk.id,
+ )
+ for observation in observations
+ ]
+ )
# ================================================
# ------------ Triplet Extraction ----------------
@@ -56,7 +90,7 @@ def ingest_data(self, args: dict):
kg_agent.update_kg(
information=information,
- metadata=payload.meta_keys,
+ metadata={**payload.meta_keys, "resource_id": text_chunk.id},
identification_params=payload.identification_params,
)