Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

fix(tasks): improived redis conf w/ memory limits + featured retrieval of cache payload

Christianf7efd10a0096
diff --git a/README.md b/README.md
index c40ff10..7daec5e 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
<p align="center">
- <img src="https://img.shields.io/badge/version-2.1.1--dev-blue?style=for-the-badge" alt="Version"/>
+ <img src="https://img.shields.io/badge/version-2.1.2--dev-blue?style=for-the-badge" alt="Version"/>
<img src="https://img.shields.io/badge/python-3.11+-green?style=for-the-badge&logo=python&logoColor=white" alt="Python"/>
<img src="https://img.shields.io/badge/license-AGPLv3%20%2B%20Commons%20Clause-purple?style=for-the-badge" alt="License"/>
</p>
diff --git a/example-docker-compose.yaml b/example-docker-compose.yaml
index a3f2d3a..c7b2903 100644
--- a/example-docker-compose.yaml
+++ b/example-docker-compose.yaml
@@ -12,12 +12,12 @@ services:
redis-server
--appendonly yes
--maxmemory 2gb
- --maxmemory-policy allkeys-lru
+ --maxmemory-policy volatile-lru
--save 900 1
--save 300 10
--save 60 10000
--tcp-backlog 511
- --timeout 300
+ --timeout 0
networks:
- appnet
restart: unless-stopped
@@ -223,10 +223,10 @@ services:
-A src.workers.app worker
--loglevel=info
--concurrency=10
- --max-tasks-per-child=1000
- --time-limit=300
- --soft-time-limit=240
- --prefetch-multiplier=2
+ --max-tasks-per-child=50
+ --time-limit=3600
+ --soft-time-limit=3000
+ --prefetch-multiplier=1
--optimization=fair
depends_on:
redis:
diff --git a/pyproject.toml b/pyproject.toml
index f5a0626..d32002a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "brainapi2"
-version = "2.1.1-dev"
+version = "2.1.2-dev"
description = "Version 2.x.x of the BrainAPI memory layer."
authors = [
{name = "Christian",email = "[email protected]"}
diff --git a/src/services/api/routes/ingest.py b/src/services/api/routes/ingest.py
index 20d65b8..e0f72cb 100644
--- a/src/services/api/routes/ingest.py
+++ b/src/services/api/routes/ingest.py
@@ -50,9 +50,9 @@ async def ingest_data(data: IngestionRequestBody):
cache_adapter.set(
key=f"task:{task.id}",
- value=json.dumps({"status": "processing", "task_id": task.id}),
+ value=json.dumps({"status": "queued", "task_id": task.id}),
brain_id=data.brain_id,
- expires_in=3600 * 24,
+ expires_in=3600 * 24 * 7,
)
return JSONResponse(
@@ -77,9 +77,9 @@ async def ingest_structured_data(data: IngestionStructuredRequestBody):
cache_adapter.set(
key=f"task:{task.id}",
- value=json.dumps({"status": "processing", "task_id": task.id}),
+ value=json.dumps({"status": "queued", "task_id": task.id}),
brain_id=data.brain_id,
- expires_in=3600 * 24,
+ expires_in=3600 * 24 * 7,
)
return JSONResponse(
diff --git a/src/services/api/routes/tasks.py b/src/services/api/routes/tasks.py
index 7d02d99..1d4cee4 100644
--- a/src/services/api/routes/tasks.py
+++ b/src/services/api/routes/tasks.py
@@ -27,7 +27,11 @@ async def get_tasks(brain_id: str = "default"):
continue
result = json.loads(str_result)
results.append(
- {"id": task.split(":")[-1], "status": result.get("status", "unknown")}
+ {
+ **result,
+ "id": task.split(":")[-1],
+ "status": result.get("status", "unknown"),
+ }
)
return {"tasks": results}
except Exception as e:
@@ -56,9 +60,9 @@ async def get_task(task_id: str, brain_id: str = "default"):
else:
result = json.loads(str_result)
return {
+ **result,
"task_id": task_id,
"status": result.get("status", "unknown"),
- "result": result,
}
except Exception as e:
log(f"Error in get_task: {type(e).__name__}: {str(e)}")
diff --git a/src/workers/app.py b/src/workers/app.py
index 0d10962..3713860 100644
--- a/src/workers/app.py
+++ b/src/workers/app.py
@@ -43,4 +43,13 @@ ingestion_app.conf.update(
broker_pool_limit=100,
broker_connection_retry_on_startup=True,
broker_connection_timeout=10,
+ result_expires=3600 * 24 * 7,
+ task_track_started=True,
+ task_time_limit=3600,
+ task_soft_time_limit=3000,
+ broker_transport_options={
+ "visibility_timeout": 7200,
+ "fanout_prefix": True,
+ "fanout_patterns": True,
+ },
)
diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.py
index d733890..836990c 100644
--- a/src/workers/tasks/ingestion.py
+++ b/src/workers/tasks/ingestion.py
@@ -64,9 +64,17 @@ def ingest_data(self, args: dict):
"""
Ingest data into the database.
"""
+ payload = None
try:
payload = IngestionTaskArgs(**args)
+ cache_adapter.set(
+ key=f"task:{self.request.id}",
+ value=json.dumps({"status": "started", "task_id": self.request.id}),
+ brain_id=payload.brain_id,
+ expires_in=3600 * 24 * 7,
+ )
+
payload.meta_keys = (
{
f"{k.replace(' ', '_').lower()}": v
@@ -144,23 +152,24 @@ def ingest_data(self, args: dict):
key=f"task:{self.request.id}",
value=json.dumps({"status": "completed", "task_id": self.request.id}),
brain_id=payload.brain_id,
- expires_in=3600 * 24,
+ expires_in=3600 * 24 * 7,
)
return self.request.id
except Exception as e:
+ brain_id = payload.brain_id if payload else args.get("brain_id", "default")
error_result = {
"status": "failed",
"task_id": self.request.id,
"error": str(e),
- "payload": payload.model_dump(mode="json"),
+ "payload": payload.model_dump(mode="json") if payload else args,
}
cache_adapter.set(
key=f"task:{self.request.id}",
value=json.dumps(error_result),
- brain_id=payload.brain_id,
+ brain_id=brain_id,
expires_in=3600 * 24 * 7,
)
@@ -172,9 +181,23 @@ def ingest_structured_data(self, args: dict):
"""
Ingest structured data into the database.
"""
+ payload = None
try:
payload = IngestionStructuredRequestBody(**args)
+ cache_adapter.set(
+ key=f"task:{self.request.id}",
+ value=json.dumps(
+ {
+ "status": "started",
+ "task_id": self.request.id,
+ "total_elements": len(payload.data),
+ }
+ ),
+ brain_id=payload.brain_id,
+ expires_in=3600 * 24 * 7,
+ )
+
for element in payload.data:
uuid = str(uuid4())
description = llm_small_adapter.generate_text(
@@ -315,23 +338,24 @@ def ingest_structured_data(self, args: dict):
key=f"task:{self.request.id}",
value=json.dumps({"status": "completed", "task_id": self.request.id}),
brain_id=payload.brain_id,
- expires_in=3600 * 24,
+ expires_in=3600 * 24 * 7,
)
return self.request.id
except Exception as e:
+ brain_id = payload.brain_id if payload else args.get("brain_id", "default")
error_result = {
"status": "failed",
"task_id": self.request.id,
"error": str(e),
- "payload": payload.model_dump(mode="json"),
+ "payload": payload.model_dump(mode="json") if payload else args,
}
cache_adapter.set(
key=f"task:{self.request.id}",
value=json.dumps(error_result),
- brain_id=payload.brain_id,
+ brain_id=brain_id,
expires_in=3600 * 24 * 7,
)