BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
misc(): fixed common node neighbors + various build issues for prod
Christian5e81a8b43ddd
diff --git a/README.md b/README.mdindex e69de29..da448a1 100644--- a/README.md+++ b/README.md@@ -0,0 +1,87 @@+## New deploy config++```+sudo apt update+sudo apt install -y ca-certificates curl gnupg lsb-release+sudo mkdir -p /etc/apt/keyrings+curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | \+sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg+echo \+ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \+ https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \+ $(lsb_release -cs) stable" | \+ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null+sudo apt update+sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin+sudo systemctl enable docker+sudo systemctl start docker+sudo docker pull nginx:latest+docker network create appnet || true+sudo docker run -d \+ --network appnet \+ --name nginx-server \+ -p 80:80 \+ -v /srv/nginx/html:/usr/share/nginx/html \+ -v /srv/nginx/conf:/etc/nginx/conf.d \+ --restart always \+ nginx+sudo systemctl enable docker+sudo docker run -it --rm \+ -v /etc/letsencrypt:/etc/letsencrypt \+ -v /var/lib/letsencrypt:/var/lib/letsencrypt \+ -p 80:80 \+ certbot/certbot certonly --standalone \+ -d glo-matcher.brainapi.lumen-labs.ai+sudo tee /srv/nginx/conf/default.conf >/dev/null <<'NGINX'+server {+ listen 80;+ server_name glo-matcher.brainapi.lumen-labs.ai;+ return 301 https://$host$request_uri;+}+server {+ listen 443 ssl;+ server_name glo-matcher.brainapi.lumen-labs.ai;++ ssl_certificate /etc/letsencrypt/live/glo-matcher.brainapi.lumen-labs.ai/fullchain.pem;+ ssl_certificate_key /etc/letsencrypt/live/glo-matcher.brainapi.lumen-labs.ai/privkey.pem;++ client_max_body_size 50m;++ location / {+ proxy_pass http://127.0.0.1:8000;+ proxy_http_version 1.1;+ proxy_set_header Host $host;+ proxy_set_header X-Real-IP $remote_addr;+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;+ proxy_set_header X-Forwarded-Proto $scheme;++ proxy_set_header Upgrade $http_upgrade;+ proxy_set_header Connection "upgrade";++ proxy_connect_timeout 60s;+ proxy_send_timeout 300s;+ proxy_read_timeout 300s;+ send_timeout 300s;+ }+}+NGINX++docker rm -f nginx-server 2>/dev/null || true+docker run -d --name nginx-server \+ -p 80:80 -p 443:443 \+ -v /srv/nginx/html:/usr/share/nginx/html \+ -v /srv/nginx/conf:/etc/nginx/conf.d \+ -v /etc/letsencrypt:/etc/letsencrypt:ro \+ --restart always \+ nginx:latest++echo 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 | docker login ghcr.io -u ChrisCoder9000 --password-stdin+docker pull ghcr.io/lumen-labs/brainapi:latest+docker run -d \+ --name brainapi \+ -p 8000:8000 \+ --network appnet \+ --restart always \+ ghcr.io/lumen-labs/brainapi:latest++```diff --git a/docker-compose-partial.yaml b/docker-compose-partial.yamlnew file mode 100644index 0000000..3d873ff--- /dev/null+++ b/docker-compose-partial.yaml@@ -0,0 +1,123 @@+services:+ redis:+ image: redis:latest+ ports:+ - "6379:6379"+ volumes:+ - redis-data:/data+ command: redis-server --appendonly yes+ networks:+ - appnet+ restart: unless-stopped+ healthcheck:+ test: ["CMD", "redis-cli", "ping"]+ interval: 10s+ timeout: 3s+ retries: 3++ neo4j:+ image: neo4j:latest+ ports:+ - "7474:7474"+ - "7687:7687"+ volumes:+ - neo4j-data:/data+ - neo4j-logs:/logs+ - neo4j-plugins:/plugins+ - ./src/lib/neo4j/config:/var/lib/neo4j/conf+ environment:+ - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD:-your_password}+ - NEO4J_PLUGINS=["apoc"]+ networks:+ - appnet+ restart: unless-stopped+ healthcheck:+ test: ["CMD-SHELL", "wget -q --spider http://localhost:7474 || exit 1"]+ interval: 15s+ timeout: 10s+ retries: 5+ start_period: 90s++ brainapi:+ image: ghcr.io/lumen-labs/brainapi:latest+ ports:+ - "8000:8000"+ env_file:+ - /root/.env+ environment:+ - REDIS_HOST=redis+ - REDIS_PORT=6379+ - NEO4J_HOST=neo4j+ - NEO4J_PORT=7687+ - NEO4J_USERNAME=neo4j+ - NEO4J_PASSWORD=${NEO4J_PASSWORD:-your_password}+ depends_on:+ redis:+ condition: service_healthy+ neo4j:+ condition: service_started+ networks:+ - appnet+ restart: unless-stopped+ healthcheck:+ test: ["CMD", "curl", "-f", "http://localhost:8000/docs"]+ interval: 30s+ timeout: 10s+ retries: 3+ start_period: 40s++ brainapi-worker:+ image: ghcr.io/lumen-labs/brainapi:latest+ command: /app/.venv/bin/celery -A src.workers.app worker --loglevel=info+ env_file:+ - /root/.env+ environment:+ - REDIS_HOST=redis+ - REDIS_PORT=6379+ - NEO4J_HOST=neo4j+ - NEO4J_PORT=7687+ - NEO4J_USERNAME=neo4j+ - NEO4J_PASSWORD=${NEO4J_PASSWORD:-your_password}+ depends_on:+ redis:+ condition: service_healthy+ neo4j:+ condition: service_started+ networks:+ - appnet+ restart: unless-stopped++ nginx:+ image: nginx:latest+ ports:+ - "80:80"+ - "443:443"+ volumes:+ - /srv/nginx/conf:/etc/nginx/conf.d:ro+ - /etc/letsencrypt:/etc/letsencrypt:ro+ - /srv/nginx/html:/usr/share/nginx/html:ro+ depends_on:+ brainapi:+ condition: service_healthy+ networks:+ - appnet+ restart: unless-stopped+ healthcheck:+ test: ["CMD", "nginx", "-t"]+ interval: 30s+ timeout: 5s+ retries: 3++networks:+ appnet:+ driver: bridge++volumes:+ redis-data:+ driver: local+ neo4j-data:+ driver: local+ neo4j-logs:+ driver: local+ neo4j-plugins:+ driver: local\ No newline at end of filediff --git a/poetry.lock b/poetry.lockindex 62512ee..6427edf 100644--- a/poetry.lock+++ b/poetry.lock@@ -3758,4 +3758,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt[metadata]lock-version = "2.1"python-versions = ">=3.11,<4.0.0"-content-hash = "b8f5bd4625042937e907cb79df3a8f1af12e67dcb3dcbb5e52c9afe1b4c4003e"+content-hash = "e70f050a323d4e9cfd509c70e99c0a4d63db3b5475f1b6f7d3c446eb7fa56197"diff --git a/pyproject.toml b/pyproject.tomlindex 06ff1ca..c724ed4 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "brainapi2"-version = "1.0.0-dev"+version = "1.1.4-dev"description = "Version 1.x.x of the BrainAPI memory layer."authors = [{name = "Christian",email = "[email protected]"}@@ -50,6 +50,7 @@ langchain-openai = ">=1.0.0,<2.0.0"google-genai = ">=1.46.0,<2.0.0"langchain-google-vertexai = ">=3.0.1,<4.0.0"pymilvus = ">=2.6.2,<3.0.0"+levenshtein = ">=0.27.1,<0.28.0"[tool.pylint.messages_control]enable = ["abstract-method"]diff --git a/src/config.py b/src/config.pyindex 73a9d93..81c6c9b 100644--- a/src/config.py+++ b/src/config.py@@ -49,7 +49,8 @@ class RedisConfig:def __init__(self):self.host = os.getenv("REDIS_HOST")- self.port = os.getenv("REDIS_PORT")+ port_str = os.getenv("REDIS_PORT")+ self.port = int(port_str) if port_str else Noneif [self.host, self.port].count(None) > 0:raise ValueError("Redis configuration is not complete")diff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.pyindex 382df22..c61841d 100644--- a/src/lib/neo4j/client.py+++ b/src/lib/neo4j/client.py@@ -415,7 +415,8 @@ class Neo4jClient(GraphClient):MATCH (n {{uuid: '{node.uuid}'}})-[r]-(c)-[r2]-(m)WHERE size([l IN labels(n) WHERE l IN labels(m)]) > 0RETURN m.uuid AS uuid, m.name AS name, labels(m) AS labels, m.description AS description, properties(m) AS properties, r2 AS rel,- CASE WHEN startNode(r2) = c THEN 'out' ELSE 'in' END AS direction+ CASE WHEN startNode(r2) = c THEN 'out' ELSE 'in' END AS direction,+ c.uuid AS c_uuid, c.name AS c_name, labels(c) AS c_labels, c.description AS c_description, properties(c) AS c_properties"""result = self.driver.execute_query(cypher_query)@@ -448,11 +449,11 @@ class Neo4jClient(GraphClient):direction=direction,),Node(- uuid=node.uuid,- name=node.name,- labels=node.labels,- description=node.description,- properties=node.properties,+ uuid=record["c_uuid"],+ name=record["c_name"],+ labels=record["c_labels"],+ description=record["c_description"],+ properties=record["c_properties"],),))diff --git a/src/lib/redis/client.py b/src/lib/redis/client.pyindex faedb2a..8b8d939 100644--- a/src/lib/redis/client.py+++ b/src/lib/redis/client.py@@ -9,6 +9,7 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@"""from redis import Redis+from redis.connection import ConnectionPoolfrom src.adapters.interfaces.cache import CacheClientfrom src.config import config@@ -19,7 +20,16 @@ class RedisClient(CacheClient):"""def __init__(self):- self.client = Redis(host=config.redis.host, port=config.redis.port)+ pool = ConnectionPool(+ host=config.redis.host,+ port=config.redis.port,+ socket_connect_timeout=5,+ socket_timeout=5,+ retry_on_timeout=True,+ health_check_interval=30,+ max_connections=50,+ )+ self.client = Redis(connection_pool=pool)def get(self, key: str) -> str:"""diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.pyindex b6d7b93..71fce9d 100644--- a/src/services/api/constants/requests.py+++ b/src/services/api/constants/requests.py@@ -8,7 +8,6 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@-----"""-from turtle import stfrom typing import List, Any, Optionalfrom pydantic import BaseModel, Field, field_serializerfrom src.constants.data import Observation, TextChunkdiff --git a/src/services/api/controllers/retrieve.py b/src/services/api/controllers/retrieve.pyindex d405c3c..518bd67 100644--- a/src/services/api/controllers/retrieve.py+++ b/src/services/api/controllers/retrieve.py@@ -40,10 +40,8 @@ async def retrieve_data(preferred_entities = []def _get_data():- st1 = time.time()text_embeddings = embeddings_adapter.embed_text(text)- et1 = time.time()- print("[time1]", et1 - st1)+data_vectors = vector_store_adapter.search_vectors(text_embeddings.embeddings, "data", limit)@@ -65,7 +63,6 @@ async def retrieve_data(for tv in triple_vectorsfor node_id in tv.metadata.get("node_ids", [])]- print("[preferred entities]", preferred_entities, type(preferred_entities))nodes = graph_adapter.get_nodes_by_uuid(uuids=node_ids,with_relationships=True,@@ -116,7 +113,7 @@ async def retrieve_neighbors(raw = graph_adapter.get_neighbors(node, limit)normalized: List[RetrievedNeighborNode] = []- print("[found direct neighbors]", len(raw))+for item in raw:if isinstance(item, tuple) and len(item) >= 2:n, pred, common = item@@ -216,27 +213,29 @@ async def retrieve_neighbors(neighbor_nodes = graph_adapter.get_by_uuids(uuids=[vector.metadata.get("uuid") for vector in v],)- normalized.extend(- RetrievedNeighborNode(- neighbor=Node(**n.model_dump()),- relationship=Predicate(- name="<SIMILAR_TO>",- description=f"<SYS_DESCRIPTION>The neighbor and the most common are similar. Most common is directly connected to {node.name}.</SYS_DESCRIPTION>",- direction="neutral",- level="2",- ),- most_common=related_nodes[k],- observations=[],- )- for n in neighbor_nodes+ for n in neighbor_nodes:+ if len(normalized) >= limit:+ breakif not any([norm.neighbor.uuid == n.uuidor norm.most_common.uuid == related_nodes[k].uuidfor norm in normalized]- )- )+ ):+ normalized.append(+ RetrievedNeighborNode(+ neighbor=Node(**n.model_dump()),+ relationship=Predicate(+ name="<SIMILAR_TO>",+ description=f"<SYS_DESCRIPTION>The neighbor and the most common are similar. Most common is directly connected to {node.name}.</SYS_DESCRIPTION>",+ direction="neutral",+ level="2",+ ),+ most_common=related_nodes[k],+ observations=[],+ )+ )# Getting the connected nodes of the similar nodes and filtering# the ones the are with same label as the main node@@ -246,20 +245,22 @@ async def retrieve_neighbors(limit=limit - len(normalized),with_labels=node.labels,)- normalized.extend(- RetrievedNeighborNode(- neighbor=Node(**n.model_dump()),- relationship=Predicate(- name=pred.name,- description=pred.description,- direction=pred.direction,- level="3",- ),- most_common=Node(**common.model_dump()),- )- for n, pred, common in neighbor_nodes- if not any([norm.neighbor.uuid == n.uuid for norm in normalized])- )+ for n, pred, common in neighbor_nodes:+ if len(normalized) >= limit:+ break+ if not any([norm.neighbor.uuid == n.uuid for norm in normalized]):+ normalized.append(+ RetrievedNeighborNode(+ neighbor=Node(**n.model_dump()),+ relationship=Predicate(+ name=pred.name,+ description=pred.description,+ direction=pred.direction,+ level="3",+ ),+ most_common=Node(**common.model_dump()),+ )+ )# if len(normalized) < limit:# v_data = vector_store_adapter.search_similar_by_ids(@@ -294,12 +295,10 @@ async def retrieve_neighbors_ai_mode(node = graph_adapter.get_by_identification_params(identification_params, entity_types=identification_params.entity_types)- print("[node]", node)if not node:raise HTTPException(status_code=404, detail="Entity not found")result = kg_agent.retrieve_neighbors(node, looking_for, limit)- print("[result]", result)ids = [neighbor.uuid for neighbor in result.neighbors]descriptions = [neighbor.description for neighbor in result.neighbors]diff --git a/src/services/webserver/nginx/nginx.conf b/src/services/webserver/nginx/nginx.confindex 3532f6f..77fdce1 100644--- a/src/services/webserver/nginx/nginx.conf+++ b/src/services/webserver/nginx/nginx.conf@@ -1,64 +1,31 @@-worker_processes auto;--events {- worker_connections 1024;-}--http {- include /etc/nginx/mime.types;- default_type application/octet-stream;-- # Logs- access_log /var/log/nginx/access.log;- error_log /var/log/nginx/error.log warn;-- # Basic hardening & sane buffers- sendfile on;- tcp_nopush on;- tcp_nodelay on;- keepalive_timeout 65;- types_hash_max_size 4096;- server_tokens off;- client_max_body_size 10m;-- # Gzip (optional; enable if you want)- # gzip on;- # gzip_types text/plain text/css application/json application/javascript application/xml+rss;-- # --- Server on port 8000 ---- server {- listen 8000 default_server;- listen [::]:8000 default_server;-- # Static files root (mounted from ./html)- root /usr/share/nginx/html;- index index.html;-- # Simple health endpoint- location = /healthz {- return 200 "ok\n";- add_header Content-Type text/plain;- }-- # Try files, then 404- location / {- try_files $uri $uri/ =404;- }-- # Example: uncomment to proxy a backend instead of static files- # upstream app_upstream { server host.docker.internal:8001; } # macOS/Windows Docker- # upstream app_upstream { server 172.17.0.1:8001; } # common Linux host- #- # location /api/ {- # proxy_pass http://app_upstream;- # proxy_set_header Host $host;- # proxy_set_header X-Real-IP $remote_addr;- # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;- # proxy_set_header X-Forwarded-Proto $scheme;- # }-- # Error pages (optional)- error_page 404 /404.html;- location = /404.html { internal; }- }+server {+ listen 80;+ server_name {}.brainapi.lumen-labs.ai;+ return 301 https://$host$request_uri;}+server {+ listen 443 ssl;+ server_name {}.brainapi.lumen-labs.ai;++ ssl_certificate /etc/letsencrypt/live/{}.brainapi.lumen-labs.ai/fullchain.pem;+ ssl_certificate_key /etc/letsencrypt/live/{}.brainapi.lumen-labs.ai/privkey.pem;++ client_max_body_size 50m;++ location / {+ proxy_pass http://127.0.0.1:8000;+ proxy_http_version 1.1;+ proxy_set_header Host $host;+ proxy_set_header X-Real-IP $remote_addr;+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;+ proxy_set_header X-Forwarded-Proto $scheme;++ proxy_set_header Upgrade $http_upgrade;+ proxy_set_header Connection "upgrade";++ proxy_connect_timeout 60s;+ proxy_send_timeout 300s;+ proxy_read_timeout 300s;+ send_timeout 300s;+ }+}\ No newline at end of filediff --git a/src/workers/app.py b/src/workers/app.pyindex 97fe21c..400f702 100644--- a/src/workers/app.py+++ b/src/workers/app.py@@ -11,18 +11,20 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@import osfrom celery import Celery+REDIS_HOST = os.getenv("REDIS_HOST", "localhost")+REDIS_PORT = os.getenv("REDIS_PORT", "6379")ingestion_app = Celery("ingestion_app",broker=("amqp://kalo:kalo@localhost:5672/"if os.getenv("CELERY_BACKEND") == "rabbitmq"- else "redis://localhost:6379/0"+ else f"redis://{REDIS_HOST}:{REDIS_PORT}/0"),backend=("rpc://"if os.getenv("CELERY_BACKEND") == "rabbitmq"- else "redis://localhost:6379/0"+ else f"redis://{REDIS_HOST}:{REDIS_PORT}/0"),include=["src.workers.tasks.ingestion"],)