Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

misc(): fixed common node neighbors + various build issues for prod

Christian5e81a8b43ddd
diff --git a/README.md b/README.md
index e69de29..da448a1 100644
--- a/README.md
+++ b/README.md
@@ -0,0 +1,87 @@
+## New deploy config
+
+```
+sudo apt update
+sudo apt install -y ca-certificates curl gnupg lsb-release
+sudo mkdir -p /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | \
+sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+echo \
+ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
+ https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \
+ $(lsb_release -cs) stable" | \
+ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt update
+sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+sudo systemctl enable docker
+sudo systemctl start docker
+sudo docker pull nginx:latest
+docker network create appnet || true
+sudo docker run -d \
+ --network appnet \
+ --name nginx-server \
+ -p 80:80 \
+ -v /srv/nginx/html:/usr/share/nginx/html \
+ -v /srv/nginx/conf:/etc/nginx/conf.d \
+ --restart always \
+ nginx
+sudo systemctl enable docker
+sudo docker run -it --rm \
+ -v /etc/letsencrypt:/etc/letsencrypt \
+ -v /var/lib/letsencrypt:/var/lib/letsencrypt \
+ -p 80:80 \
+ certbot/certbot certonly --standalone \
+ -d glo-matcher.brainapi.lumen-labs.ai
+sudo tee /srv/nginx/conf/default.conf >/dev/null <<'NGINX'
+server {
+ listen 80;
+ server_name glo-matcher.brainapi.lumen-labs.ai;
+ return 301 https://$host$request_uri;
+}
+server {
+ listen 443 ssl;
+ server_name glo-matcher.brainapi.lumen-labs.ai;
+
+ ssl_certificate /etc/letsencrypt/live/glo-matcher.brainapi.lumen-labs.ai/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/glo-matcher.brainapi.lumen-labs.ai/privkey.pem;
+
+ client_max_body_size 50m;
+
+ location / {
+ proxy_pass http://127.0.0.1:8000;
+ proxy_http_version 1.1;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+
+ proxy_connect_timeout 60s;
+ proxy_send_timeout 300s;
+ proxy_read_timeout 300s;
+ send_timeout 300s;
+ }
+}
+NGINX
+
+docker rm -f nginx-server 2>/dev/null || true
+docker run -d --name nginx-server \
+ -p 80:80 -p 443:443 \
+ -v /srv/nginx/html:/usr/share/nginx/html \
+ -v /srv/nginx/conf:/etc/nginx/conf.d \
+ -v /etc/letsencrypt:/etc/letsencrypt:ro \
+ --restart always \
+ nginx:latest
+
+echo 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 | docker login ghcr.io -u ChrisCoder9000 --password-stdin
+docker pull ghcr.io/lumen-labs/brainapi:latest
+docker run -d \
+ --name brainapi \
+ -p 8000:8000 \
+ --network appnet \
+ --restart always \
+ ghcr.io/lumen-labs/brainapi:latest
+
+```
diff --git a/docker-compose-partial.yaml b/docker-compose-partial.yaml
new file mode 100644
index 0000000..3d873ff
--- /dev/null
+++ b/docker-compose-partial.yaml
@@ -0,0 +1,123 @@
+services:
+ redis:
+ image: redis:latest
+ ports:
+ - "6379:6379"
+ volumes:
+ - redis-data:/data
+ command: redis-server --appendonly yes
+ networks:
+ - appnet
+ restart: unless-stopped
+ healthcheck:
+ test: ["CMD", "redis-cli", "ping"]
+ interval: 10s
+ timeout: 3s
+ retries: 3
+
+ neo4j:
+ image: neo4j:latest
+ ports:
+ - "7474:7474"
+ - "7687:7687"
+ volumes:
+ - neo4j-data:/data
+ - neo4j-logs:/logs
+ - neo4j-plugins:/plugins
+ - ./src/lib/neo4j/config:/var/lib/neo4j/conf
+ environment:
+ - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD:-your_password}
+ - NEO4J_PLUGINS=["apoc"]
+ networks:
+ - appnet
+ restart: unless-stopped
+ healthcheck:
+ test: ["CMD-SHELL", "wget -q --spider http://localhost:7474 || exit 1"]
+ interval: 15s
+ timeout: 10s
+ retries: 5
+ start_period: 90s
+
+ brainapi:
+ image: ghcr.io/lumen-labs/brainapi:latest
+ ports:
+ - "8000:8000"
+ env_file:
+ - /root/.env
+ environment:
+ - REDIS_HOST=redis
+ - REDIS_PORT=6379
+ - NEO4J_HOST=neo4j
+ - NEO4J_PORT=7687
+ - NEO4J_USERNAME=neo4j
+ - NEO4J_PASSWORD=${NEO4J_PASSWORD:-your_password}
+ depends_on:
+ redis:
+ condition: service_healthy
+ neo4j:
+ condition: service_started
+ networks:
+ - appnet
+ restart: unless-stopped
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:8000/docs"]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ start_period: 40s
+
+ brainapi-worker:
+ image: ghcr.io/lumen-labs/brainapi:latest
+ command: /app/.venv/bin/celery -A src.workers.app worker --loglevel=info
+ env_file:
+ - /root/.env
+ environment:
+ - REDIS_HOST=redis
+ - REDIS_PORT=6379
+ - NEO4J_HOST=neo4j
+ - NEO4J_PORT=7687
+ - NEO4J_USERNAME=neo4j
+ - NEO4J_PASSWORD=${NEO4J_PASSWORD:-your_password}
+ depends_on:
+ redis:
+ condition: service_healthy
+ neo4j:
+ condition: service_started
+ networks:
+ - appnet
+ restart: unless-stopped
+
+ nginx:
+ image: nginx:latest
+ ports:
+ - "80:80"
+ - "443:443"
+ volumes:
+ - /srv/nginx/conf:/etc/nginx/conf.d:ro
+ - /etc/letsencrypt:/etc/letsencrypt:ro
+ - /srv/nginx/html:/usr/share/nginx/html:ro
+ depends_on:
+ brainapi:
+ condition: service_healthy
+ networks:
+ - appnet
+ restart: unless-stopped
+ healthcheck:
+ test: ["CMD", "nginx", "-t"]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+
+networks:
+ appnet:
+ driver: bridge
+
+volumes:
+ redis-data:
+ driver: local
+ neo4j-data:
+ driver: local
+ neo4j-logs:
+ driver: local
+ neo4j-plugins:
+ driver: local
\ No newline at end of file
diff --git a/poetry.lock b/poetry.lock
index 62512ee..6427edf 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -3758,4 +3758,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<4.0.0"
-content-hash = "b8f5bd4625042937e907cb79df3a8f1af12e67dcb3dcbb5e52c9afe1b4c4003e"
+content-hash = "e70f050a323d4e9cfd509c70e99c0a4d63db3b5475f1b6f7d3c446eb7fa56197"
diff --git a/pyproject.toml b/pyproject.toml
index 06ff1ca..c724ed4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "brainapi2"
-version = "1.0.0-dev"
+version = "1.1.4-dev"
description = "Version 1.x.x of the BrainAPI memory layer."
authors = [
{name = "Christian",email = "[email protected]"}
@@ -50,6 +50,7 @@ langchain-openai = ">=1.0.0,<2.0.0"
google-genai = ">=1.46.0,<2.0.0"
langchain-google-vertexai = ">=3.0.1,<4.0.0"
pymilvus = ">=2.6.2,<3.0.0"
+levenshtein = ">=0.27.1,<0.28.0"
[tool.pylint.messages_control]
enable = ["abstract-method"]
diff --git a/src/config.py b/src/config.py
index 73a9d93..81c6c9b 100644
--- a/src/config.py
+++ b/src/config.py
@@ -49,7 +49,8 @@ class RedisConfig:
def __init__(self):
self.host = os.getenv("REDIS_HOST")
- self.port = os.getenv("REDIS_PORT")
+ port_str = os.getenv("REDIS_PORT")
+ self.port = int(port_str) if port_str else None
if [self.host, self.port].count(None) > 0:
raise ValueError("Redis configuration is not complete")
diff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.py
index 382df22..c61841d 100644
--- a/src/lib/neo4j/client.py
+++ b/src/lib/neo4j/client.py
@@ -415,7 +415,8 @@ class Neo4jClient(GraphClient):
MATCH (n {{uuid: '{node.uuid}'}})-[r]-(c)-[r2]-(m)
WHERE size([l IN labels(n) WHERE l IN labels(m)]) > 0
RETURN m.uuid AS uuid, m.name AS name, labels(m) AS labels, m.description AS description, properties(m) AS properties, r2 AS rel,
- CASE WHEN startNode(r2) = c THEN 'out' ELSE 'in' END AS direction
+ CASE WHEN startNode(r2) = c THEN 'out' ELSE 'in' END AS direction,
+ c.uuid AS c_uuid, c.name AS c_name, labels(c) AS c_labels, c.description AS c_description, properties(c) AS c_properties
"""
result = self.driver.execute_query(cypher_query)
@@ -448,11 +449,11 @@ class Neo4jClient(GraphClient):
direction=direction,
),
Node(
- uuid=node.uuid,
- name=node.name,
- labels=node.labels,
- description=node.description,
- properties=node.properties,
+ uuid=record["c_uuid"],
+ name=record["c_name"],
+ labels=record["c_labels"],
+ description=record["c_description"],
+ properties=record["c_properties"],
),
)
)
diff --git a/src/lib/redis/client.py b/src/lib/redis/client.py
index faedb2a..8b8d939 100644
--- a/src/lib/redis/client.py
+++ b/src/lib/redis/client.py
@@ -9,6 +9,7 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
"""
from redis import Redis
+from redis.connection import ConnectionPool
from src.adapters.interfaces.cache import CacheClient
from src.config import config
@@ -19,7 +20,16 @@ class RedisClient(CacheClient):
"""
def __init__(self):
- self.client = Redis(host=config.redis.host, port=config.redis.port)
+ pool = ConnectionPool(
+ host=config.redis.host,
+ port=config.redis.port,
+ socket_connect_timeout=5,
+ socket_timeout=5,
+ retry_on_timeout=True,
+ health_check_interval=30,
+ max_connections=50,
+ )
+ self.client = Redis(connection_pool=pool)
def get(self, key: str) -> str:
"""
diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.py
index b6d7b93..71fce9d 100644
--- a/src/services/api/constants/requests.py
+++ b/src/services/api/constants/requests.py
@@ -8,7 +8,6 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
-----
"""
-from turtle import st
from typing import List, Any, Optional
from pydantic import BaseModel, Field, field_serializer
from src.constants.data import Observation, TextChunk
diff --git a/src/services/api/controllers/retrieve.py b/src/services/api/controllers/retrieve.py
index d405c3c..518bd67 100644
--- a/src/services/api/controllers/retrieve.py
+++ b/src/services/api/controllers/retrieve.py
@@ -40,10 +40,8 @@ async def retrieve_data(
preferred_entities = []
def _get_data():
- st1 = time.time()
text_embeddings = embeddings_adapter.embed_text(text)
- et1 = time.time()
- print("[time1]", et1 - st1)
+
data_vectors = vector_store_adapter.search_vectors(
text_embeddings.embeddings, "data", limit
)
@@ -65,7 +63,6 @@ async def retrieve_data(
for tv in triple_vectors
for node_id in tv.metadata.get("node_ids", [])
]
- print("[preferred entities]", preferred_entities, type(preferred_entities))
nodes = graph_adapter.get_nodes_by_uuid(
uuids=node_ids,
with_relationships=True,
@@ -116,7 +113,7 @@ async def retrieve_neighbors(
raw = graph_adapter.get_neighbors(node, limit)
normalized: List[RetrievedNeighborNode] = []
- print("[found direct neighbors]", len(raw))
+
for item in raw:
if isinstance(item, tuple) and len(item) >= 2:
n, pred, common = item
@@ -216,27 +213,29 @@ async def retrieve_neighbors(
neighbor_nodes = graph_adapter.get_by_uuids(
uuids=[vector.metadata.get("uuid") for vector in v],
)
- normalized.extend(
- RetrievedNeighborNode(
- neighbor=Node(**n.model_dump()),
- relationship=Predicate(
- name="<SIMILAR_TO>",
- description=f"<SYS_DESCRIPTION>The neighbor and the most common are similar. Most common is directly connected to {node.name}.</SYS_DESCRIPTION>",
- direction="neutral",
- level="2",
- ),
- most_common=related_nodes[k],
- observations=[],
- )
- for n in neighbor_nodes
+ for n in neighbor_nodes:
+ if len(normalized) >= limit:
+ break
if not any(
[
norm.neighbor.uuid == n.uuid
or norm.most_common.uuid == related_nodes[k].uuid
for norm in normalized
]
- )
- )
+ ):
+ normalized.append(
+ RetrievedNeighborNode(
+ neighbor=Node(**n.model_dump()),
+ relationship=Predicate(
+ name="<SIMILAR_TO>",
+ description=f"<SYS_DESCRIPTION>The neighbor and the most common are similar. Most common is directly connected to {node.name}.</SYS_DESCRIPTION>",
+ direction="neutral",
+ level="2",
+ ),
+ most_common=related_nodes[k],
+ observations=[],
+ )
+ )
# Getting the connected nodes of the similar nodes and filtering
# the ones the are with same label as the main node
@@ -246,20 +245,22 @@ async def retrieve_neighbors(
limit=limit - len(normalized),
with_labels=node.labels,
)
- normalized.extend(
- RetrievedNeighborNode(
- neighbor=Node(**n.model_dump()),
- relationship=Predicate(
- name=pred.name,
- description=pred.description,
- direction=pred.direction,
- level="3",
- ),
- most_common=Node(**common.model_dump()),
- )
- for n, pred, common in neighbor_nodes
- if not any([norm.neighbor.uuid == n.uuid for norm in normalized])
- )
+ for n, pred, common in neighbor_nodes:
+ if len(normalized) >= limit:
+ break
+ if not any([norm.neighbor.uuid == n.uuid for norm in normalized]):
+ normalized.append(
+ RetrievedNeighborNode(
+ neighbor=Node(**n.model_dump()),
+ relationship=Predicate(
+ name=pred.name,
+ description=pred.description,
+ direction=pred.direction,
+ level="3",
+ ),
+ most_common=Node(**common.model_dump()),
+ )
+ )
# if len(normalized) < limit:
# v_data = vector_store_adapter.search_similar_by_ids(
@@ -294,12 +295,10 @@ async def retrieve_neighbors_ai_mode(
node = graph_adapter.get_by_identification_params(
identification_params, entity_types=identification_params.entity_types
)
- print("[node]", node)
if not node:
raise HTTPException(status_code=404, detail="Entity not found")
result = kg_agent.retrieve_neighbors(node, looking_for, limit)
- print("[result]", result)
ids = [neighbor.uuid for neighbor in result.neighbors]
descriptions = [neighbor.description for neighbor in result.neighbors]
diff --git a/src/services/webserver/nginx/nginx.conf b/src/services/webserver/nginx/nginx.conf
index 3532f6f..77fdce1 100644
--- a/src/services/webserver/nginx/nginx.conf
+++ b/src/services/webserver/nginx/nginx.conf
@@ -1,64 +1,31 @@
-worker_processes auto;
-
-events {
- worker_connections 1024;
-}
-
-http {
- include /etc/nginx/mime.types;
- default_type application/octet-stream;
-
- # Logs
- access_log /var/log/nginx/access.log;
- error_log /var/log/nginx/error.log warn;
-
- # Basic hardening & sane buffers
- sendfile on;
- tcp_nopush on;
- tcp_nodelay on;
- keepalive_timeout 65;
- types_hash_max_size 4096;
- server_tokens off;
- client_max_body_size 10m;
-
- # Gzip (optional; enable if you want)
- # gzip on;
- # gzip_types text/plain text/css application/json application/javascript application/xml+rss;
-
- # --- Server on port 8000 ---
- server {
- listen 8000 default_server;
- listen [::]:8000 default_server;
-
- # Static files root (mounted from ./html)
- root /usr/share/nginx/html;
- index index.html;
-
- # Simple health endpoint
- location = /healthz {
- return 200 "ok\n";
- add_header Content-Type text/plain;
- }
-
- # Try files, then 404
- location / {
- try_files $uri $uri/ =404;
- }
-
- # Example: uncomment to proxy a backend instead of static files
- # upstream app_upstream { server host.docker.internal:8001; } # macOS/Windows Docker
- # upstream app_upstream { server 172.17.0.1:8001; } # common Linux host
- #
- # location /api/ {
- # proxy_pass http://app_upstream;
- # proxy_set_header Host $host;
- # proxy_set_header X-Real-IP $remote_addr;
- # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- # proxy_set_header X-Forwarded-Proto $scheme;
- # }
-
- # Error pages (optional)
- error_page 404 /404.html;
- location = /404.html { internal; }
- }
+server {
+ listen 80;
+ server_name {}.brainapi.lumen-labs.ai;
+ return 301 https://$host$request_uri;
}
+server {
+ listen 443 ssl;
+ server_name {}.brainapi.lumen-labs.ai;
+
+ ssl_certificate /etc/letsencrypt/live/{}.brainapi.lumen-labs.ai/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/{}.brainapi.lumen-labs.ai/privkey.pem;
+
+ client_max_body_size 50m;
+
+ location / {
+ proxy_pass http://127.0.0.1:8000;
+ proxy_http_version 1.1;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+
+ proxy_connect_timeout 60s;
+ proxy_send_timeout 300s;
+ proxy_read_timeout 300s;
+ send_timeout 300s;
+ }
+}
\ No newline at end of file
diff --git a/src/workers/app.py b/src/workers/app.py
index 97fe21c..400f702 100644
--- a/src/workers/app.py
+++ b/src/workers/app.py
@@ -11,18 +11,20 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
import os
from celery import Celery
+REDIS_HOST = os.getenv("REDIS_HOST", "localhost")
+REDIS_PORT = os.getenv("REDIS_PORT", "6379")
ingestion_app = Celery(
"ingestion_app",
broker=(
"amqp://kalo:kalo@localhost:5672/"
if os.getenv("CELERY_BACKEND") == "rabbitmq"
- else "redis://localhost:6379/0"
+ else f"redis://{REDIS_HOST}:{REDIS_PORT}/0"
),
backend=(
"rpc://"
if os.getenv("CELERY_BACKEND") == "rabbitmq"
- else "redis://localhost:6379/0"
+ else f"redis://{REDIS_HOST}:{REDIS_PORT}/0"
),
include=["src.workers.tasks.ingestion"],
)