BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
fix(ingestion): fixed redis ingestion
Christianfeb0b959e5bf
diff --git a/README.md b/README.mdindex aa9e2fb..c40ff10 100644--- a/README.md+++ b/README.md@@ -1,5 +1,5 @@<p align="center">- <img src="https://img.shields.io/badge/version-2.1.0--dev-blue?style=for-the-badge" alt="Version"/>+ <img src="https://img.shields.io/badge/version-2.1.1--dev-blue?style=for-the-badge" alt="Version"/><img src="https://img.shields.io/badge/python-3.11+-green?style=for-the-badge&logo=python&logoColor=white" alt="Python"/><img src="https://img.shields.io/badge/license-AGPLv3%20%2B%20Commons%20Clause-purple?style=for-the-badge" alt="License"/></p>diff --git a/pyproject.toml b/pyproject.tomlindex 665b919..f5a0626 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "brainapi2"-version = "2.1.0-dev"+version = "2.1.1-dev"description = "Version 2.x.x of the BrainAPI memory layer."authors = [{name = "Christian",email = "[email protected]"}diff --git a/src/core/saving/auto_kg.py b/src/core/saving/auto_kg.pyindex f641ca8..844af19 100644--- a/src/core/saving/auto_kg.py+++ b/src/core/saving/auto_kg.py@@ -153,7 +153,7 @@ def enrich_kg_from_input(for future in all_futures:try:- normalized_result = future.result(timeout=90)+ normalized_result = future.result(timeout=180)input_tokens += normalized_result.input_tokens or 0output_tokens += normalized_result.output_tokens or 0if normalized_result.entity:@@ -184,7 +184,7 @@ def enrich_kg_from_input(for future in relationship_futures:try:- normalized_result = future.result(timeout=90)+ normalized_result = future.result(timeout=180)input_tokens += normalized_result.input_tokens or 0output_tokens += normalized_result.output_tokens or 0if normalized_result.relationship:@@ -215,7 +215,7 @@ def enrich_kg_from_input(for future, node_data in node_embedding_futures:try:- future.result(timeout=60)+ future.result(timeout=180)graph_nodes.append(Node(uuid=node_data.uuid,diff --git a/src/services/api/routes/ingest.py b/src/services/api/routes/ingest.pyindex 62c42b0..20d65b8 100644--- a/src/services/api/routes/ingest.py+++ b/src/services/api/routes/ingest.py@@ -9,11 +9,13 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@"""import json-from fastapi import APIRouter+import asyncio+from fastapi import APIRouter, HTTPExceptionfrom typing import Literalfrom typing_extensions import Annotatedfrom fastapi import Form, UploadFilefrom starlette.responses import JSONResponse+from celery.exceptions import OperationalErrorfrom src.services.api.constants.requests import (IngestionRequestBody,@@ -25,6 +27,9 @@ from src.workers.tasks.ingestion import (ingest_structured_data as ingest_structured_data_task,)+MAX_TASK_RETRIES = 3+RETRY_DELAY_BASE = 0.1+ingest_router = APIRouter(prefix="/ingest", tags=["ingest"])@@ -33,7 +38,15 @@ async def ingest_data(data: IngestionRequestBody):"""Ingest data to the processing pipeline and save to the memory."""- task = ingest_data_task.delay(data.model_dump())+ task = None+ for attempt in range(MAX_TASK_RETRIES):+ try:+ task = ingest_data_task.delay(data.model_dump())+ break+ except OperationalError:+ if attempt == MAX_TASK_RETRIES - 1:+ raise HTTPException(status_code=503, detail="Task queue unavailable")+ await asyncio.sleep(RETRY_DELAY_BASE * (attempt + 1))cache_adapter.set(key=f"task:{task.id}",@@ -52,7 +65,15 @@ async def ingest_structured_data(data: IngestionStructuredRequestBody):"""Ingest structured data to the processing pipeline and save to the memory."""- task = ingest_structured_data_task.delay(data.model_dump())+ task = None+ for attempt in range(MAX_TASK_RETRIES):+ try:+ task = ingest_structured_data_task.delay(data.model_dump())+ break+ except OperationalError:+ if attempt == MAX_TASK_RETRIES - 1:+ raise HTTPException(status_code=503, detail="Task queue unavailable")+ await asyncio.sleep(RETRY_DELAY_BASE * (attempt + 1))cache_adapter.set(key=f"task:{task.id}",diff --git a/src/workers/app.py b/src/workers/app.pyindex 5f1a9a9..0d10962 100644--- a/src/workers/app.py+++ b/src/workers/app.py@@ -40,4 +40,7 @@ ingestion_app.conf.update(task_reject_on_worker_lost=True,worker_pool="threads",worker_concurrency=config.celery.worker_concurrency,+ broker_pool_limit=100,+ broker_connection_retry_on_startup=True,+ broker_connection_timeout=10,)