BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
fix(tasks): improived redis conf w/ memory limits + featured retrieval of cache payload
Christianf7efd10a0096
diff --git a/README.md b/README.mdindex c40ff10..7daec5e 100644--- a/README.md+++ b/README.md@@ -1,5 +1,5 @@<p align="center">- <img src="https://img.shields.io/badge/version-2.1.1--dev-blue?style=for-the-badge" alt="Version"/>+ <img src="https://img.shields.io/badge/version-2.1.2--dev-blue?style=for-the-badge" alt="Version"/><img src="https://img.shields.io/badge/python-3.11+-green?style=for-the-badge&logo=python&logoColor=white" alt="Python"/><img src="https://img.shields.io/badge/license-AGPLv3%20%2B%20Commons%20Clause-purple?style=for-the-badge" alt="License"/></p>diff --git a/example-docker-compose.yaml b/example-docker-compose.yamlindex a3f2d3a..c7b2903 100644--- a/example-docker-compose.yaml+++ b/example-docker-compose.yaml@@ -12,12 +12,12 @@ services:redis-server--appendonly yes--maxmemory 2gb- --maxmemory-policy allkeys-lru+ --maxmemory-policy volatile-lru--save 900 1--save 300 10--save 60 10000--tcp-backlog 511- --timeout 300+ --timeout 0networks:- appnetrestart: unless-stopped@@ -223,10 +223,10 @@ services:-A src.workers.app worker--loglevel=info--concurrency=10- --max-tasks-per-child=1000- --time-limit=300- --soft-time-limit=240- --prefetch-multiplier=2+ --max-tasks-per-child=50+ --time-limit=3600+ --soft-time-limit=3000+ --prefetch-multiplier=1--optimization=fairdepends_on:redis:diff --git a/pyproject.toml b/pyproject.tomlindex f5a0626..d32002a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "brainapi2"-version = "2.1.1-dev"+version = "2.1.2-dev"description = "Version 2.x.x of the BrainAPI memory layer."authors = [{name = "Christian",email = "[email protected]"}diff --git a/src/services/api/routes/ingest.py b/src/services/api/routes/ingest.pyindex 20d65b8..e0f72cb 100644--- a/src/services/api/routes/ingest.py+++ b/src/services/api/routes/ingest.py@@ -50,9 +50,9 @@ async def ingest_data(data: IngestionRequestBody):cache_adapter.set(key=f"task:{task.id}",- value=json.dumps({"status": "processing", "task_id": task.id}),+ value=json.dumps({"status": "queued", "task_id": task.id}),brain_id=data.brain_id,- expires_in=3600 * 24,+ expires_in=3600 * 24 * 7,)return JSONResponse(@@ -77,9 +77,9 @@ async def ingest_structured_data(data: IngestionStructuredRequestBody):cache_adapter.set(key=f"task:{task.id}",- value=json.dumps({"status": "processing", "task_id": task.id}),+ value=json.dumps({"status": "queued", "task_id": task.id}),brain_id=data.brain_id,- expires_in=3600 * 24,+ expires_in=3600 * 24 * 7,)return JSONResponse(diff --git a/src/services/api/routes/tasks.py b/src/services/api/routes/tasks.pyindex 7d02d99..1d4cee4 100644--- a/src/services/api/routes/tasks.py+++ b/src/services/api/routes/tasks.py@@ -27,7 +27,11 @@ async def get_tasks(brain_id: str = "default"):continueresult = json.loads(str_result)results.append(- {"id": task.split(":")[-1], "status": result.get("status", "unknown")}+ {+ **result,+ "id": task.split(":")[-1],+ "status": result.get("status", "unknown"),+ })return {"tasks": results}except Exception as e:@@ -56,9 +60,9 @@ async def get_task(task_id: str, brain_id: str = "default"):else:result = json.loads(str_result)return {+ **result,"task_id": task_id,"status": result.get("status", "unknown"),- "result": result,}except Exception as e:log(f"Error in get_task: {type(e).__name__}: {str(e)}")diff --git a/src/workers/app.py b/src/workers/app.pyindex 0d10962..3713860 100644--- a/src/workers/app.py+++ b/src/workers/app.py@@ -43,4 +43,13 @@ ingestion_app.conf.update(broker_pool_limit=100,broker_connection_retry_on_startup=True,broker_connection_timeout=10,+ result_expires=3600 * 24 * 7,+ task_track_started=True,+ task_time_limit=3600,+ task_soft_time_limit=3000,+ broker_transport_options={+ "visibility_timeout": 7200,+ "fanout_prefix": True,+ "fanout_patterns": True,+ },)diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.pyindex d733890..836990c 100644--- a/src/workers/tasks/ingestion.py+++ b/src/workers/tasks/ingestion.py@@ -64,9 +64,17 @@ def ingest_data(self, args: dict):"""Ingest data into the database."""+ payload = Nonetry:payload = IngestionTaskArgs(**args)+ cache_adapter.set(+ key=f"task:{self.request.id}",+ value=json.dumps({"status": "started", "task_id": self.request.id}),+ brain_id=payload.brain_id,+ expires_in=3600 * 24 * 7,+ )+payload.meta_keys = ({f"{k.replace(' ', '_').lower()}": v@@ -144,23 +152,24 @@ def ingest_data(self, args: dict):key=f"task:{self.request.id}",value=json.dumps({"status": "completed", "task_id": self.request.id}),brain_id=payload.brain_id,- expires_in=3600 * 24,+ expires_in=3600 * 24 * 7,)return self.request.idexcept Exception as e:+ brain_id = payload.brain_id if payload else args.get("brain_id", "default")error_result = {"status": "failed","task_id": self.request.id,"error": str(e),- "payload": payload.model_dump(mode="json"),+ "payload": payload.model_dump(mode="json") if payload else args,}cache_adapter.set(key=f"task:{self.request.id}",value=json.dumps(error_result),- brain_id=payload.brain_id,+ brain_id=brain_id,expires_in=3600 * 24 * 7,)@@ -172,9 +181,23 @@ def ingest_structured_data(self, args: dict):"""Ingest structured data into the database."""+ payload = Nonetry:payload = IngestionStructuredRequestBody(**args)+ cache_adapter.set(+ key=f"task:{self.request.id}",+ value=json.dumps(+ {+ "status": "started",+ "task_id": self.request.id,+ "total_elements": len(payload.data),+ }+ ),+ brain_id=payload.brain_id,+ expires_in=3600 * 24 * 7,+ )+for element in payload.data:uuid = str(uuid4())description = llm_small_adapter.generate_text(@@ -315,23 +338,24 @@ def ingest_structured_data(self, args: dict):key=f"task:{self.request.id}",value=json.dumps({"status": "completed", "task_id": self.request.id}),brain_id=payload.brain_id,- expires_in=3600 * 24,+ expires_in=3600 * 24 * 7,)return self.request.idexcept Exception as e:+ brain_id = payload.brain_id if payload else args.get("brain_id", "default")error_result = {"status": "failed","task_id": self.request.id,"error": str(e),- "payload": payload.model_dump(mode="json"),+ "payload": payload.model_dump(mode="json") if payload else args,}cache_adapter.set(key=f"task:{self.request.id}",value=json.dumps(error_result),- brain_id=payload.brain_id,+ brain_id=brain_id,expires_in=3600 * 24 * 7,)