Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

feat(workers): - parallelized celery tasks into owned queues

Christian32637968cfe4
diff --git a/example-docker-compose.yaml b/example-docker-compose.yaml
index c7b2903..142b035 100644
--- a/example-docker-compose.yaml
+++ b/example-docker-compose.yaml
@@ -220,7 +220,8 @@ services:
- .env
command: >
/app/.venv/bin/celery
- -A src.workers.app worker
+ -A src.workers.app:ingestion_app worker
+ -Q ingest_data,process_architect_relationships,ingest_structured_data,consolidate_graph
--loglevel=info
--concurrency=10
--max-tasks-per-child=50
@@ -248,7 +249,7 @@ services:
cpus: "0.9"
memory: 1408m
healthcheck:
- test: ["CMD", "celery", "-A", "src.workers.app", "inspect", "ping"]
+ test: ["CMD", "celery", "-A", "src.workers.app:ingestion_app", "inspect", "ping"]
interval: 30s
timeout: 5s
retries: 3
diff --git a/src/workers/app.py b/src/workers/app.py
index 8fb0627..ad1f254 100644
--- a/src/workers/app.py
+++ b/src/workers/app.py
@@ -17,6 +17,7 @@ _project_root = Path(__file__).resolve().parent.parent.parent
dotenv.load_dotenv(_project_root / ".env")
from celery import Celery
+from kombu import Queue
from src.config import config
@@ -40,13 +41,37 @@ ingestion_app = Celery(
include=["src.workers.tasks.ingestion"],
)
+TASK_QUEUES = (
+ Queue("ingest_data", routing_key="ingest_data"),
+ Queue(
+ "process_architect_relationships", routing_key="process_architect_relationships"
+ ),
+ Queue("ingest_structured_data", routing_key="ingest_structured_data"),
+ Queue("consolidate_graph", routing_key="consolidate_graph"),
+)
+
+TASK_ROUTES = {
+ "src.workers.tasks.ingestion.ingest_data": {"queue": "ingest_data"},
+ "src.workers.tasks.ingestion.process_architect_relationships": {
+ "queue": "process_architect_relationships"
+ },
+ "src.workers.tasks.ingestion.ingest_structured_data": {
+ "queue": "ingest_structured_data"
+ },
+ "src.workers.tasks.ingestion.consolidate_graph_async": {
+ "queue": "consolidate_graph"
+ },
+}
+
ingestion_app.conf.update(
+ task_queues=TASK_QUEUES,
+ task_routes=TASK_ROUTES,
worker_max_tasks_per_child=50,
worker_prefetch_multiplier=1,
task_acks_late=True,
task_reject_on_worker_lost=True,
worker_pool="threads",
- worker_concurrency=config.celery.worker_concurrency,
+ worker_concurrency=int(config.celery.worker_concurrency),
broker_pool_limit=100,
broker_connection_retry_on_startup=True,
broker_connection_timeout=10,