BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
feat(workers): - parallelized celery tasks into owned queues
Christian32637968cfe4
diff --git a/example-docker-compose.yaml b/example-docker-compose.yamlindex c7b2903..142b035 100644--- a/example-docker-compose.yaml+++ b/example-docker-compose.yaml@@ -220,7 +220,8 @@ services:- .envcommand: >/app/.venv/bin/celery- -A src.workers.app worker+ -A src.workers.app:ingestion_app worker+ -Q ingest_data,process_architect_relationships,ingest_structured_data,consolidate_graph--loglevel=info--concurrency=10--max-tasks-per-child=50@@ -248,7 +249,7 @@ services:cpus: "0.9"memory: 1408mhealthcheck:- test: ["CMD", "celery", "-A", "src.workers.app", "inspect", "ping"]+ test: ["CMD", "celery", "-A", "src.workers.app:ingestion_app", "inspect", "ping"]interval: 30stimeout: 5sretries: 3diff --git a/src/workers/app.py b/src/workers/app.pyindex 8fb0627..ad1f254 100644--- a/src/workers/app.py+++ b/src/workers/app.py@@ -17,6 +17,7 @@ _project_root = Path(__file__).resolve().parent.parent.parentdotenv.load_dotenv(_project_root / ".env")from celery import Celery+from kombu import Queuefrom src.config import config@@ -40,13 +41,37 @@ ingestion_app = Celery(include=["src.workers.tasks.ingestion"],)+TASK_QUEUES = (+ Queue("ingest_data", routing_key="ingest_data"),+ Queue(+ "process_architect_relationships", routing_key="process_architect_relationships"+ ),+ Queue("ingest_structured_data", routing_key="ingest_structured_data"),+ Queue("consolidate_graph", routing_key="consolidate_graph"),+)++TASK_ROUTES = {+ "src.workers.tasks.ingestion.ingest_data": {"queue": "ingest_data"},+ "src.workers.tasks.ingestion.process_architect_relationships": {+ "queue": "process_architect_relationships"+ },+ "src.workers.tasks.ingestion.ingest_structured_data": {+ "queue": "ingest_structured_data"+ },+ "src.workers.tasks.ingestion.consolidate_graph_async": {+ "queue": "consolidate_graph"+ },+}+ingestion_app.conf.update(+ task_queues=TASK_QUEUES,+ task_routes=TASK_ROUTES,worker_max_tasks_per_child=50,worker_prefetch_multiplier=1,task_acks_late=True,task_reject_on_worker_lost=True,worker_pool="threads",- worker_concurrency=config.celery.worker_concurrency,+ worker_concurrency=int(config.celery.worker_concurrency),broker_pool_limit=100,broker_connection_retry_on_startup=True,broker_connection_timeout=10,