Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(temporal): Wave 11.F — per-plugin activities + worker pools
trohitg620a4984b08d
diff --git a/server/services/temporal/plugin_activities.py b/server/services/temporal/plugin_activities.pynew file mode 100644index 00000000..684df78b--- /dev/null+++ b/server/services/temporal/plugin_activities.py@@ -0,0 +1,109 @@+"""Wave 11.F: per-plugin Temporal activities.++Walks :func:`services.node_registry.registered_node_classes` and exposes+each plugin's ``cls.as_activity()`` as a Temporal activity. Workers can+register these alongside (or instead of) the single+``NodeExecutionActivities.execute_node_activity`` to get per-node+scaling knobs:++- ``cls.task_queue`` routes activities to specialised worker pools+ (``rest-api`` / ``ai-heavy`` / ``code-exec`` / ``triggers-poll`` /+ ``triggers-event`` / ``android`` / ``browser`` / ``messaging`` /+ ``machina-default``).+- ``cls.retry_policy`` per-node Temporal retry knobs.+- ``cls.start_to_close_timeout`` + ``cls.heartbeat_timeout`` per-node.++Activity name convention: ``node.{type}.v{version}``.++Wire-up:+ from services.temporal.plugin_activities import collect_plugin_activities+ activities = collect_plugin_activities()+ worker = Worker(client, task_queue=..., activities=activities, ...)++:func:`collect_plugin_activities_for_queue` returns only the activities+whose plugin declares a matching ``task_queue`` — use this when running+a specialised worker pool.+"""++from __future__ import annotations++from typing import Any, Callable, Dict, Iterable, List, Optional++from core.logging import get_logger++logger = get_logger(__name__)+++def collect_plugin_activities(+ *,+ task_queue: Optional[str] = None,+ include_types: Optional[Iterable[str]] = None,+) -> List[Callable]:+ """Return ``@activity.defn``-decorated callables for every plugin+ class registered via :func:`register_node_class`.++ Args:+ task_queue: if given, only include activities whose plugin's+ ``cls.task_queue`` matches this value. Use this to build+ specialised worker pools.+ include_types: optional whitelist of node-type strings. Useful+ for tests and incremental rollout.++ Returns:+ A list of Temporal activity callables ready to pass to+ ``Worker(activities=[...])``.+ """+ # Lazy import so this module can be imported before nodes have been+ # discovered (e.g. during worker bootstrap).+ from services.node_registry import registered_node_classes++ activities: List[Callable] = []+ classes = registered_node_classes()+ for node_type, cls in classes.items():+ if include_types is not None and node_type not in include_types:+ continue+ cls_queue = getattr(cls, "task_queue", None)+ if task_queue is not None and cls_queue != task_queue:+ continue+ try:+ activity_fn = cls.as_activity()+ except Exception as e:+ logger.warning(+ "Failed to build activity for %s: %s", node_type, e,+ )+ continue+ activities.append(activity_fn)++ logger.info(+ "Collected %d plugin activities%s",+ len(activities),+ f" for queue={task_queue!r}" if task_queue else "",+ )+ return activities+++def collect_plugin_activities_for_queue(task_queue: str) -> List[Callable]:+ """Convenience wrapper for the single-queue case."""+ return collect_plugin_activities(task_queue=task_queue)+++def distinct_task_queues() -> List[str]:+ """Return every task_queue string declared by any registered plugin.++ Lets worker-pool orchestration discover which queues need workers+ without hardcoding the list.+ """+ from services.node_registry import registered_node_classes++ seen: Dict[str, int] = {}+ for cls in registered_node_classes().values():+ q = getattr(cls, "task_queue", None)+ if q:+ seen[q] = seen.get(q, 0) + 1+ queues = sorted(seen.keys())+ logger.info(+ "Plugins declare %d distinct task queues: %s",+ len(queues),+ {q: seen[q] for q in queues},+ )+ return queuesdiff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.pyindex 17da53b8..9383e898 100644--- a/server/services/temporal/worker.py+++ b/server/services/temporal/worker.py@@ -92,13 +92,32 @@ class TemporalWorkerManager:# Create activity instance with shared sessionself._activities = NodeExecutionActivities(self._session)+ # Wave 11.F: register per-plugin activities alongside the+ # legacy generic execute_node_activity. Workers pick up both.+ # The legacy activity stays so existing workflows that call+ # "execute_node_activity" keep working; plugin activities are+ # indexed by ``node.{type}.v{version}`` for future per-node+ # dispatch.+ activities: list = [self._activities.execute_node_activity]+ try:+ import nodes # populate plugin-class registry+ from services.temporal.plugin_activities import collect_plugin_activities+ plugin_activities = collect_plugin_activities()+ activities.extend(plugin_activities)+ logger.info(+ "Registered %d plugin activities on queue %s",+ len(plugin_activities), self.task_queue,+ )+ except Exception as e:+ logger.warning(f"Plugin activity registration failed: {e}")+# Create worker with class-based activity# For class-based activities, pass the bound method (instance.method)self._worker = Worker(self.client,task_queue=self.task_queue,workflows=[MachinaWorkflow],- activities=[self._activities.execute_node_activity], # Pass bound method+ activities=activities,# Allow concurrent activity execution for parallel branchesmax_concurrent_activities=self.pool_size,max_concurrent_workflow_tasks=10,@@ -152,6 +171,106 @@ class TemporalWorkerManager:logger.info("Temporal worker stopped")+class TemporalWorkerPool:+ """Wave 11.F: multi-queue worker pool.++ Runs one :class:`Worker` per declared plugin ``task_queue``. Each+ worker only registers activities whose plugin opts in to that+ queue, so specialised workloads can be scaled independently.+ Env overrides for per-pool concurrency: ``TEMPORAL_<QUEUE>_CONCURRENCY``+ (e.g. ``TEMPORAL_AI_HEAVY_CONCURRENCY=4``).++ Usage::++ pool = TemporalWorkerPool(client, queues=["rest-api", "ai-heavy"])+ await pool.start()+ try:+ ...+ finally:+ await pool.stop()+ """++ # Default concurrency per queue — override via env.+ DEFAULT_CONCURRENCY: dict[str, int] = {+ "machina-default": 20,+ "rest-api": 50,+ "ai-heavy": 4,+ "code-exec": 10,+ "triggers-poll": 100,+ "triggers-event": 100,+ "android": 10,+ "browser": 4,+ "messaging": 20,+ }++ def __init__(+ self,+ client: Client,+ *,+ queues: Optional[list[str]] = None,+ default_pool_size: int = 20,+ ):+ self.client = client+ self.default_pool_size = default_pool_size+ # Lazy import to avoid circular at module load time.+ from services.temporal.plugin_activities import distinct_task_queues++ self.queues = queues or distinct_task_queues()+ self._workers: list[Worker] = []+ self._tasks: list[asyncio.Task] = []++ def _concurrency_for(self, queue: str) -> int:+ import os++ env_key = f"TEMPORAL_{queue.upper().replace('-', '_')}_CONCURRENCY"+ raw = os.environ.get(env_key)+ if raw and raw.isdigit():+ return int(raw)+ return self.DEFAULT_CONCURRENCY.get(queue, self.default_pool_size)++ @property+ def is_running(self) -> bool:+ return any(t is not None and not t.done() for t in self._tasks)++ async def start(self) -> None:+ from services.temporal.plugin_activities import (+ collect_plugin_activities,+ )++ for queue in self.queues:+ activities = collect_plugin_activities(task_queue=queue)+ if not activities:+ logger.info(f"[Pool] Skipping empty queue {queue!r}")+ continue+ concurrency = self._concurrency_for(queue)+ worker = Worker(+ self.client,+ task_queue=queue,+ activities=activities,+ max_concurrent_activities=concurrency,+ max_concurrent_workflow_tasks=10,+ )+ task = asyncio.create_task(worker.run(), name=f"worker-{queue}")+ self._workers.append(worker)+ self._tasks.append(task)+ logger.info(+ f"[Pool] Started worker queue={queue!r} "+ f"activities={len(activities)} concurrency={concurrency}"+ )++ async def stop(self) -> None:+ for task in self._tasks:+ task.cancel()+ for task in self._tasks:+ try:+ await task+ except (asyncio.CancelledError, Exception):+ pass+ self._workers.clear()+ self._tasks.clear()+ logger.info("[Pool] All workers stopped")++async def run_standalone_worker(server_address: str = "localhost:7233",namespace: str = "default",