Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

feat(tasks): improved redis persistence

Christian67be7b03e540
diff --git a/src/adapters/cache.py b/src/adapters/cache.py
index 33dba1b..20ff7b2 100644
--- a/src/adapters/cache.py
+++ b/src/adapters/cache.py
@@ -55,3 +55,9 @@ class CacheAdapter:
Get all task keys for a given brain_id.
"""
return self.cache.get_task_keys(brain_id)
+
+ def get_task(self, task_id: str, brain_id: str = "default") -> Optional[str]:
+ """
+ Get a specific task by ID.
+ """
+ return self.cache.get_task(task_id, brain_id)
diff --git a/src/adapters/interfaces/cache.py b/src/adapters/interfaces/cache.py
index 202dae2..8d97a07 100644
--- a/src/adapters/interfaces/cache.py
+++ b/src/adapters/interfaces/cache.py
@@ -54,3 +54,10 @@ class CacheClient(ABC):
Get all task keys for a given brain_id.
"""
raise NotImplementedError("get_task_keys method not implemented")
+
+ @abstractmethod
+ def get_task(self, task_id: str, brain_id: str) -> Optional[str]:
+ """
+ Get a specific task by ID.
+ """
+ raise NotImplementedError("get_task method not implemented")
diff --git a/src/lib/redis/client.py b/src/lib/redis/client.py
index 7156b27..dff8fa8 100644
--- a/src/lib/redis/client.py
+++ b/src/lib/redis/client.py
@@ -8,12 +8,16 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
-----
"""
+import json
+import time
from typing import Optional
from redis import Redis
from redis.connection import ConnectionPool
from src.adapters.interfaces.cache import CacheClient
from src.config import config
+TASK_RETENTION_SECONDS = 3600 * 24 * 7
+
class RedisClient(CacheClient):
"""
@@ -55,10 +59,17 @@ class RedisClient(CacheClient):
) -> bool:
"""
Set a value in the cache with an expiration time.
+ Task keys are stored WITHOUT TTL to prevent eviction by volatile-lru.
"""
prefixed_key = self._get_key(key, brain_id)
if key.startswith("task:"):
- self.client.sadd(f"{brain_id}:_task_index", key)
+ task_id = key.split(":")[-1]
+ self.client.hset(
+ f"{brain_id}:_tasks",
+ task_id,
+ json.dumps({"data": value, "created_at": time.time()}),
+ )
+ return True
return self.client.set(
prefixed_key, value, **({"ex": expires_in} if expires_in else {})
)
@@ -69,15 +80,52 @@ class RedisClient(CacheClient):
"""
prefixed_key = self._get_key(key, brain_id)
if key.startswith("task:"):
- self.client.srem(f"{brain_id}:_task_index", key)
+ task_id = key.split(":")[-1]
+ return self.client.hdel(f"{brain_id}:_tasks", task_id)
return self.client.delete(prefixed_key)
def get_task_keys(self, brain_id: str) -> list[str]:
"""
Get all task keys for a given brain_id.
+ Also cleans up expired tasks (older than TASK_RETENTION_SECONDS).
"""
- keys = self.client.smembers(f"{brain_id}:_task_index")
- return [k.decode("utf-8") if isinstance(k, bytes) else k for k in keys]
+ tasks_hash = f"{brain_id}:_tasks"
+ all_tasks = self.client.hgetall(tasks_hash)
+ valid_keys = []
+ now = time.time()
+ expired_keys = []
+
+ for task_id, task_data in all_tasks.items():
+ task_id_str = (
+ task_id.decode("utf-8") if isinstance(task_id, bytes) else task_id
+ )
+ try:
+ data = json.loads(task_data)
+ created_at = data.get("created_at", 0)
+ if now - created_at > TASK_RETENTION_SECONDS:
+ expired_keys.append(task_id_str)
+ else:
+ valid_keys.append(f"task:{task_id_str}")
+ except (json.JSONDecodeError, TypeError):
+ expired_keys.append(task_id_str)
+
+ if expired_keys:
+ self.client.hdel(tasks_hash, *expired_keys)
+
+ return valid_keys
+
+ def get_task(self, task_id: str, brain_id: str) -> Optional[str]:
+ """
+ Get a specific task by ID.
+ """
+ task_data = self.client.hget(f"{brain_id}:_tasks", task_id)
+ if task_data is None:
+ return None
+ try:
+ data = json.loads(task_data)
+ return data.get("data")
+ except (json.JSONDecodeError, TypeError):
+ return None
_redis_client = RedisClient()
diff --git a/src/services/api/routes/tasks.py b/src/services/api/routes/tasks.py
index 1d4cee4..66e9a58 100644
--- a/src/services/api/routes/tasks.py
+++ b/src/services/api/routes/tasks.py
@@ -19,17 +19,18 @@ tasks_router = APIRouter(prefix="/tasks", tags=["tasks"])
@tasks_router.get("/")
async def get_tasks(brain_id: str = "default"):
try:
- tasks = cache_adapter.get_task_keys(brain_id)
+ task_keys = cache_adapter.get_task_keys(brain_id)
results = []
- for task in tasks:
- str_result = cache_adapter.get(task, brain_id=brain_id)
+ for task_key in task_keys:
+ task_id = task_key.split(":")[-1]
+ str_result = cache_adapter.get_task(task_id, brain_id=brain_id)
if str_result is None:
continue
result = json.loads(str_result)
results.append(
{
**result,
- "id": task.split(":")[-1],
+ "id": task_id,
"status": result.get("status", "unknown"),
}
)
@@ -48,7 +49,7 @@ async def get_task(task_id: str, brain_id: str = "default"):
Get the result of a task by its ID.
"""
try:
- str_result = cache_adapter.get(f"task:{task_id}", brain_id=brain_id)
+ str_result = cache_adapter.get_task(task_id, brain_id=brain_id)
if str_result is None:
return {
"task_id": task_id,