Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

fix(ingestion): fixed redis ingestion

Christianfeb0b959e5bf
diff --git a/README.md b/README.md
index aa9e2fb..c40ff10 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
<p align="center">
- <img src="https://img.shields.io/badge/version-2.1.0--dev-blue?style=for-the-badge" alt="Version"/>
+ <img src="https://img.shields.io/badge/version-2.1.1--dev-blue?style=for-the-badge" alt="Version"/>
<img src="https://img.shields.io/badge/python-3.11+-green?style=for-the-badge&logo=python&logoColor=white" alt="Python"/>
<img src="https://img.shields.io/badge/license-AGPLv3%20%2B%20Commons%20Clause-purple?style=for-the-badge" alt="License"/>
</p>
diff --git a/pyproject.toml b/pyproject.toml
index 665b919..f5a0626 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "brainapi2"
-version = "2.1.0-dev"
+version = "2.1.1-dev"
description = "Version 2.x.x of the BrainAPI memory layer."
authors = [
{name = "Christian",email = "[email protected]"}
diff --git a/src/core/saving/auto_kg.py b/src/core/saving/auto_kg.py
index f641ca8..844af19 100644
--- a/src/core/saving/auto_kg.py
+++ b/src/core/saving/auto_kg.py
@@ -153,7 +153,7 @@ def enrich_kg_from_input(
for future in all_futures:
try:
- normalized_result = future.result(timeout=90)
+ normalized_result = future.result(timeout=180)
input_tokens += normalized_result.input_tokens or 0
output_tokens += normalized_result.output_tokens or 0
if normalized_result.entity:
@@ -184,7 +184,7 @@ def enrich_kg_from_input(
for future in relationship_futures:
try:
- normalized_result = future.result(timeout=90)
+ normalized_result = future.result(timeout=180)
input_tokens += normalized_result.input_tokens or 0
output_tokens += normalized_result.output_tokens or 0
if normalized_result.relationship:
@@ -215,7 +215,7 @@ def enrich_kg_from_input(
for future, node_data in node_embedding_futures:
try:
- future.result(timeout=60)
+ future.result(timeout=180)
graph_nodes.append(
Node(
uuid=node_data.uuid,
diff --git a/src/services/api/routes/ingest.py b/src/services/api/routes/ingest.py
index 62c42b0..20d65b8 100644
--- a/src/services/api/routes/ingest.py
+++ b/src/services/api/routes/ingest.py
@@ -9,11 +9,13 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
"""
import json
-from fastapi import APIRouter
+import asyncio
+from fastapi import APIRouter, HTTPException
from typing import Literal
from typing_extensions import Annotated
from fastapi import Form, UploadFile
from starlette.responses import JSONResponse
+from celery.exceptions import OperationalError
from src.services.api.constants.requests import (
IngestionRequestBody,
@@ -25,6 +27,9 @@ from src.workers.tasks.ingestion import (
ingest_structured_data as ingest_structured_data_task,
)
+MAX_TASK_RETRIES = 3
+RETRY_DELAY_BASE = 0.1
+
ingest_router = APIRouter(prefix="/ingest", tags=["ingest"])
@@ -33,7 +38,15 @@ async def ingest_data(data: IngestionRequestBody):
"""
Ingest data to the processing pipeline and save to the memory.
"""
- task = ingest_data_task.delay(data.model_dump())
+ task = None
+ for attempt in range(MAX_TASK_RETRIES):
+ try:
+ task = ingest_data_task.delay(data.model_dump())
+ break
+ except OperationalError:
+ if attempt == MAX_TASK_RETRIES - 1:
+ raise HTTPException(status_code=503, detail="Task queue unavailable")
+ await asyncio.sleep(RETRY_DELAY_BASE * (attempt + 1))
cache_adapter.set(
key=f"task:{task.id}",
@@ -52,7 +65,15 @@ async def ingest_structured_data(data: IngestionStructuredRequestBody):
"""
Ingest structured data to the processing pipeline and save to the memory.
"""
- task = ingest_structured_data_task.delay(data.model_dump())
+ task = None
+ for attempt in range(MAX_TASK_RETRIES):
+ try:
+ task = ingest_structured_data_task.delay(data.model_dump())
+ break
+ except OperationalError:
+ if attempt == MAX_TASK_RETRIES - 1:
+ raise HTTPException(status_code=503, detail="Task queue unavailable")
+ await asyncio.sleep(RETRY_DELAY_BASE * (attempt + 1))
cache_adapter.set(
key=f"task:{task.id}",
diff --git a/src/workers/app.py b/src/workers/app.py
index 5f1a9a9..0d10962 100644
--- a/src/workers/app.py
+++ b/src/workers/app.py
@@ -40,4 +40,7 @@ ingestion_app.conf.update(
task_reject_on_worker_lost=True,
worker_pool="threads",
worker_concurrency=config.celery.worker_concurrency,
+ broker_pool_limit=100,
+ broker_connection_retry_on_startup=True,
+ broker_connection_timeout=10,
)