Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
perf(observability): OpenTelemetry spans + Node perf_hooks for benchmarking
trohitg64a2f79fc3f8
diff --git a/scripts/dev.js b/scripts/dev.jsindex 92835980..54c8254f 100644--- a/scripts/dev.js+++ b/scripts/dev.js@@ -15,11 +15,17 @@ import {killPort,createLogger,waitForTcpPort,+ installPerfMeasureLogger,} from './utils.js';const START_TIME = Date.now();const log = createLogger(START_TIME);+// Install the perf observer once: every performance.measure entry from+// this process (including the ones emitted by waitForTcpPort) is logged+// as a structured "[perf] <name> <ms>" line. Standard Node lib only.+installPerfMeasureLogger(log);+// Parse command line argumentsconst args = process.argv.slice(2);const isDaemonMode = args.includes('--daemon');@@ -119,7 +125,10 @@ async function main() {Promise.all(readyTargets.map(async ({ name, port }) => {- const ok = await waitForTcpPort(port);+ // Pass `name` so the perf observer emits ``tcp.wait.<service>``+ // entries instead of port-numbered ones. Timing comes from the+ // observer (single source), not inline scaffolding here.+ const ok = await waitForTcpPort(port, { name: name.toLowerCase() });log(ok ? `${name} ready (port ${port})` : `${name} did not become ready (port ${port})`);return ok;})diff --git a/scripts/start.js b/scripts/start.jsindex bb82efdb..c414d4a0 100755--- a/scripts/start.js+++ b/scripts/start.js@@ -16,11 +16,17 @@ import {killPort,createLogger,waitForTcpPort,+ installPerfMeasureLogger,} from './utils.js';const START_TIME = Date.now();const log = createLogger(START_TIME);+// One PerformanceObserver per process — every performance.measure entry+// (including those from waitForTcpPort) is logged as a structured+// "[perf] <name> <ms>" line. Standard Node lib only.+installPerfMeasureLogger(log);+const args = process.argv.slice(2);const isVerbose = args.includes('--verbose') || args.includes('-v');const skipWhatsApp = args.includes('--skip-whatsapp');@@ -127,7 +133,9 @@ async function main() {function startReadyProbes() {Promise.all(readyTargets.map(async ({ name, port }) => {- const ok = await waitForTcpPort(port);+ // Pass `name` so the perf observer emits ``tcp.wait.<service>``+ // entries instead of port-numbered ones.+ const ok = await waitForTcpPort(port, { name: name.toLowerCase() });log(ok ? `${name} ready (port ${port})` : `${name} did not become ready (port ${port})`);return ok;})diff --git a/scripts/utils.js b/scripts/utils.jsindex 346db814..19044be5 100644--- a/scripts/utils.js+++ b/scripts/utils.js@@ -9,6 +9,7 @@import { readFileSync, existsSync, copyFileSync } from 'fs';import { Socket } from 'net';import { resolve, dirname } from 'path';+import { performance, PerformanceObserver } from 'node:perf_hooks';import { fileURLToPath } from 'url';import { execSync } from 'child_process';@@ -81,17 +82,50 @@ export function probeTcpPort(port, host = '127.0.0.1', timeoutMs = 500) {* ready to accept traffic — significantly more reliable and faster than* grepping stdout for ready-message patterns, which trail actual readiness* by hundreds of ms due to line-buffered stdout flush.+ *+ * Emits a Node ``performance.measure`` entry named ``tcp.wait.<name>``+ * (or ``tcp.wait.<port>`` if no name) so callers that installed a+ * ``PerformanceObserver`` (see ``installPerfMeasureLogger``) get+ * structured benchmark output without per-callsite timing scaffolding.*/export async function waitForTcpPort(port,- { intervalMs = 250, maxWaitMs = 60000, host = '127.0.0.1' } = {}+ { intervalMs = 250, maxWaitMs = 60000, host = '127.0.0.1', name } = {}) {+ const measureName = `tcp.wait.${name ?? port}`;+ const startMark = `${measureName}.start.${performance.now()}`;+ performance.mark(startMark);const start = Date.now();- while (Date.now() - start < maxWaitMs) {- if (await probeTcpPort(port, host)) return true;- await sleep(intervalMs);+ try {+ while (Date.now() - start < maxWaitMs) {+ if (await probeTcpPort(port, host)) return true;+ await sleep(intervalMs);+ }+ return false;+ } finally {+ performance.measure(measureName, startMark);+ performance.clearMarks(startMark);}- return false;+}++/**+ * Install a ``PerformanceObserver`` that logs every ``performance.measure``+ * entry through the supplied logger. This is the Node stdlib equivalent+ * of the OpenTelemetry "tracer + console exporter" pattern: instrumented+ * code calls ``performance.mark/measure``; one observer at the script+ * boundary turns those entries into structured log lines.+ *+ * Returns the observer so the caller can ``disconnect()`` if needed+ * (rarely — observers are usually script-lifetime).+ */+export function installPerfMeasureLogger(log, { prefix = '[perf]' } = {}) {+ const observer = new PerformanceObserver((items) => {+ for (const entry of items.getEntries()) {+ log(`${prefix} ${entry.name} ${Math.round(entry.duration)}ms`);+ }+ });+ observer.observe({ entryTypes: ['measure'] });+ return observer;}/**diff --git a/server/core/tracing.py b/server/core/tracing.pynew file mode 100644index 00000000..ea443019--- /dev/null+++ b/server/core/tracing.py@@ -0,0 +1,60 @@+"""OpenTelemetry tracing bootstrap.++Single initialization point for the OTel TracerProvider. Modules that+want to emit spans do::++ from opentelemetry import trace+ tracer = trace.get_tracer(__name__)++ async def something():+ with tracer.start_as_current_span("namespace.operation"):+ ...++Spans are exported via ``ConsoleSpanExporter`` so cold-start+benchmarking is grep-able from the same terminal that runs the dev+server::++ pnpm run dev 2>&1 | grep '"name":'++In production, swap ``ConsoleSpanExporter`` for an OTLP exporter+pointed at a collector (Jaeger / Tempo / Honeycomb / etc.) — purely+config, no code change at the call sites.++This file owns the only place in the codebase where the OTel SDK is+configured. Every other module uses ``trace.get_tracer(__name__)``+which is the standard library API; no project-specific abstraction.+"""++from __future__ import annotations++from opentelemetry import trace+from opentelemetry.sdk.resources import SERVICE_NAME, Resource+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import (+ BatchSpanProcessor,+ ConsoleSpanExporter,+)+++_INITIALIZED = False+++def init_tracing(service_name: str = "machinaos-backend") -> None:+ """Configure the global TracerProvider.++ Idempotent — safe to call multiple times. Subsequent calls are+ no-ops so tests / reloads do not accumulate processors.+ """+ global _INITIALIZED+ if _INITIALIZED:+ return++ provider = TracerProvider(+ resource=Resource.create({SERVICE_NAME: service_name})+ )+ # ConsoleSpanExporter writes one JSON document per span to stdout —+ # ideal for dev/benchmark mode. Span fields include ``name``,+ # ``start_time``, ``end_time``, ``attributes``, ``status``.+ provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))+ trace.set_tracer_provider(provider)+ _INITIALIZED = Truediff --git a/server/main.py b/server/main.pyindex 4d8efe20..29d7994e 100644--- a/server/main.py+++ b/server/main.py@@ -38,13 +38,19 @@ from core.container import container_startup_log("Importing settings + logging...")from core.config import Settingsfrom core.logging import configure_logging, get_logger, setup_websocket_logging, shutdown_websocket_logging+from core.tracing import init_tracing_startup_log("Importing routers...")from routers import workflow, database, maps, nodejs_compat, android, websocket, webhook, auth, twitter, google, credentials, schemas_startup_log("All imports complete")-# Initialize settings and logging+# Initialize settings, logging, and tracing.+# init_tracing() registers the OpenTelemetry TracerProvider with a+# ConsoleSpanExporter so every span emitted via+# ``trace.get_tracer(__name__).start_as_current_span(...)`` shows up in+# stdout for cold-start benchmarking.settings = Settings()configure_logging(settings)+init_tracing()logger = get_logger(__name__)# Suppress noisy loggersdiff --git a/server/pyproject.toml b/server/pyproject.tomlindex 1ad91923..ea2fada7 100644--- a/server/pyproject.toml+++ b/server/pyproject.toml@@ -27,6 +27,10 @@ dependencies = ["structlog>=24.1.0","orjson>=3.9.0",+ # Tracing / benchmarking (OpenTelemetry — industry standard)+ "opentelemetry-api>=1.27.0",+ "opentelemetry-sdk>=1.27.0",+# YAML parsing (for SKILL.md files)"pyyaml>=6.0",diff --git a/server/services/status_broadcaster.py b/server/services/status_broadcaster.pyindex 58cb9054..fb316823 100644--- a/server/services/status_broadcaster.py+++ b/server/services/status_broadcaster.py@@ -8,9 +8,12 @@ import asyncioimport orjsonfrom typing import Set, Dict, Any, Optional, Listfrom fastapi import WebSocket+from opentelemetry import trace+from core.logging import get_loggerlogger = get_logger(__name__)+tracer = trace.get_tracer(__name__)class StatusBroadcaster:@@ -133,17 +136,22 @@ class StatusBroadcaster:Each ``_refresh_*`` already swallows its own exceptions, soTaskGroup never sees one. Kept inside try/except* defensivelyin case a future refactor lets one escape.++ Wrapped in an OTel span so the cycle's wall-clock duration is+ emitted via the configured span exporter (ConsoleSpanExporter+ in dev for benchmarking, OTLP in prod)."""- try:- async with asyncio.TaskGroup() as tg:- tg.create_task(self._refresh_whatsapp_status())- tg.create_task(self._refresh_twitter_status())- tg.create_task(self._refresh_google_status())- tg.create_task(self._refresh_telegram_status())- tg.create_task(self._auto_reconnect_android_relay())- except* Exception as eg:- for exc in eg.exceptions:- logger.warning("[StatusBroadcaster] Service refresh task failed: %s", exc)+ with tracer.start_as_current_span("broadcaster.refresh_all_services"):+ try:+ async with asyncio.TaskGroup() as tg:+ tg.create_task(self._refresh_whatsapp_status())+ tg.create_task(self._refresh_twitter_status())+ tg.create_task(self._refresh_google_status())+ tg.create_task(self._refresh_telegram_status())+ tg.create_task(self._auto_reconnect_android_relay())+ except* Exception as eg:+ for exc in eg.exceptions:+ logger.warning("[StatusBroadcaster] Service refresh task failed: %s", exc)async def broadcast(self, message: Dict[str, Any]):"""Broadcast a message to all connected clients using TaskGroup.@@ -259,105 +267,117 @@ class StatusBroadcaster:"""Fetch fresh WhatsApp status from Go service and update cache.Called on client connect to ensure initial_status has accurate data.- Silently fails if WhatsApp service is unavailable.+ Silently fails if WhatsApp service is unavailable. Span emitted+ via OTel for cold-start benchmarking."""- try:- from services.whatsapp_service import get_client- import time-- client = await get_client()- status_data = await client.call("status")-- self._status["whatsapp"] = {- "connected": status_data.get("connected", False),- "has_session": status_data.get("has_session", False),- "running": status_data.get("running", False),- "pairing": status_data.get("pairing", False),- "device_id": status_data.get("device_id"),- "qr": None,- "timestamp": time.time()- }- logger.debug(f"[StatusBroadcaster] Refreshed WhatsApp status: connected={status_data.get('connected')}")- await self.broadcast({- "type": "whatsapp_status",- "data": self._status["whatsapp"],- })- except Exception as e:- # Don't fail client connection if WhatsApp service is down- logger.debug(f"[StatusBroadcaster] Could not refresh WhatsApp status: {e}")+ with tracer.start_as_current_span("broadcaster.refresh_whatsapp") as span:+ try:+ from services.whatsapp_service import get_client+ import time++ client = await get_client()+ status_data = await client.call("status")++ self._status["whatsapp"] = {+ "connected": status_data.get("connected", False),+ "has_session": status_data.get("has_session", False),+ "running": status_data.get("running", False),+ "pairing": status_data.get("pairing", False),+ "device_id": status_data.get("device_id"),+ "qr": None,+ "timestamp": time.time()+ }+ logger.debug(f"[StatusBroadcaster] Refreshed WhatsApp status: connected={status_data.get('connected')}")+ await self.broadcast({+ "type": "whatsapp_status",+ "data": self._status["whatsapp"],+ })+ span.set_attribute("connected", bool(status_data.get("connected", False)))+ except Exception as e:+ span.record_exception(e)+ # Don't fail client connection if WhatsApp service is down+ logger.debug(f"[StatusBroadcaster] Could not refresh WhatsApp status: {e}")async def _refresh_twitter_status(self):"""Fetch Twitter status from stored OAuth tokens in database.Called on client connect to check if user is authenticated with Twitter.Uses OAuth token system (matches REST endpoint and node handlers).+ Span emitted via OTel for cold-start benchmarking."""- try:- from core.container import container- auth_service = container.auth_service()-- # Get tokens from OAuth system (NOT api_key system)- tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")- if not tokens or not tokens.get("access_token"):- self._status["twitter"] = {- "connected": False,- "username": None,- "user_id": None,- "name": None,- "profile_image_url": None- }- else:- # User info is stored in the OAuth token record- email = tokens.get("email", "") # Stored as "@username"- name = tokens.get("name", "")- username = email.lstrip("@") if email.startswith("@") else email-- self._status["twitter"] = {- "connected": True,- "username": username or None,- "user_id": None,- "name": name or None,- "profile_image_url": None- }- logger.debug(f"[StatusBroadcaster] Twitter status: connected as @{username}")+ with tracer.start_as_current_span("broadcaster.refresh_twitter") as span:+ try:+ from core.container import container+ auth_service = container.auth_service()++ # Get tokens from OAuth system (NOT api_key system)+ tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")+ if not tokens or not tokens.get("access_token"):+ self._status["twitter"] = {+ "connected": False,+ "username": None,+ "user_id": None,+ "name": None,+ "profile_image_url": None+ }+ else:+ # User info is stored in the OAuth token record+ email = tokens.get("email", "") # Stored as "@username"+ name = tokens.get("name", "")+ username = email.lstrip("@") if email.startswith("@") else email++ self._status["twitter"] = {+ "connected": True,+ "username": username or None,+ "user_id": None,+ "name": name or None,+ "profile_image_url": None+ }+ logger.debug(f"[StatusBroadcaster] Twitter status: connected as @{username}")- await self.broadcast({- "type": "twitter_status",- "data": self._status["twitter"],- })- except Exception as e:- logger.debug(f"[StatusBroadcaster] Could not refresh Twitter status: {e}")+ await self.broadcast({+ "type": "twitter_status",+ "data": self._status["twitter"],+ })+ span.set_attribute("connected", bool(self._status["twitter"]["connected"]))+ except Exception as e:+ span.record_exception(e)+ logger.debug(f"[StatusBroadcaster] Could not refresh Twitter status: {e}")async def _refresh_google_status(self):"""Fetch Google Workspace status from stored OAuth tokens.Called on client connect to check if user is authenticated with Google.+ Span emitted via OTel for cold-start benchmarking."""- try:- from core.container import container- auth_service = container.auth_service()-- tokens = await auth_service.get_oauth_tokens("google", customer_id="owner")- if not tokens or not tokens.get("access_token"):- self._status["google"] = {- "connected": False,- "email": None,- "name": None,- }- else:- self._status["google"] = {- "connected": True,- "email": tokens.get("email"),- "name": tokens.get("name"),- }- logger.debug(f"[StatusBroadcaster] Google status: connected as {tokens.get('email')}")+ with tracer.start_as_current_span("broadcaster.refresh_google") as span:+ try:+ from core.container import container+ auth_service = container.auth_service()++ tokens = await auth_service.get_oauth_tokens("google", customer_id="owner")+ if not tokens or not tokens.get("access_token"):+ self._status["google"] = {+ "connected": False,+ "email": None,+ "name": None,+ }+ else:+ self._status["google"] = {+ "connected": True,+ "email": tokens.get("email"),+ "name": tokens.get("name"),+ }+ logger.debug(f"[StatusBroadcaster] Google status: connected as {tokens.get('email')}")- await self.broadcast({- "type": "google_status",- "data": self._status["google"],- })- except Exception as e:- logger.debug(f"[StatusBroadcaster] Could not refresh Google status: {e}")+ await self.broadcast({+ "type": "google_status",+ "data": self._status["google"],+ })+ span.set_attribute("connected", bool(self._status["google"]["connected"]))+ except Exception as e:+ span.record_exception(e)+ logger.debug(f"[StatusBroadcaster] Could not refresh Google status: {e}")async def _refresh_telegram_status(self):"""Refresh Telegram status from stored credentials and auto-reconnect.@@ -366,50 +386,66 @@ class StatusBroadcaster:inside TelegramService; this function just decides whether toattempt an auto-reconnect and mirrors the resulting status intothe broadcaster cache.- """- try:- from services.telegram_service import get_telegram_service- service = get_telegram_service()-- def _snapshot(connected: bool, has_token: bool) -> Dict[str, Any]:- s = service.get_status()- return {- "connected": connected,- "bot_id": s.get("bot_id") if connected else None,- "bot_username": s.get("bot_username") if connected else None,- "bot_name": s.get("bot_name") if connected else None,- "owner_chat_id": s.get("owner_chat_id") if connected else None,- "has_stored_token": has_token,- }- if service.connected:- self._status["telegram"] = _snapshot(True, True)- elif not await service.has_stored_token():- self._status["telegram"] = _snapshot(False, False)- else:- logger.info("[StatusBroadcaster] Auto-reconnecting Telegram bot...")- result = await service.connect() # service reads token from DB- ok = bool(result.get("success"))- self._status["telegram"] = _snapshot(ok, True)- if ok:- bot_username = self._status["telegram"].get("bot_username")- logger.info(f"[StatusBroadcaster] Telegram auto-reconnected: @{bot_username}")+ Span emitted via OTel for cold-start benchmarking. This is the+ canonical "is the broadcaster fast?" metric — Telegram's getMe+ on cold DNS is the historical bottleneck (10s read_timeout).+ """+ with tracer.start_as_current_span("broadcaster.refresh_telegram") as span:+ try:+ from services.telegram_service import get_telegram_service+ service = get_telegram_service()++ def _snapshot(connected: bool, has_token: bool) -> Dict[str, Any]:+ s = service.get_status()+ return {+ "connected": connected,+ "bot_id": s.get("bot_id") if connected else None,+ "bot_username": s.get("bot_username") if connected else None,+ "bot_name": s.get("bot_name") if connected else None,+ "owner_chat_id": s.get("owner_chat_id") if connected else None,+ "has_stored_token": has_token,+ }++ if service.connected:+ self._status["telegram"] = _snapshot(True, True)+ span.set_attribute("path", "already_connected")+ elif not await service.has_stored_token():+ self._status["telegram"] = _snapshot(False, False)+ span.set_attribute("path", "no_token")else:- logger.warning(f"[StatusBroadcaster] Telegram auto-reconnect failed: {result.get('error')}")+ span.set_attribute("path", "auto_reconnect")+ logger.info("[StatusBroadcaster] Auto-reconnecting Telegram bot...")+ result = await service.connect() # service reads token from DB+ ok = bool(result.get("success"))+ self._status["telegram"] = _snapshot(ok, True)+ span.set_attribute("reconnect_ok", ok)+ if ok:+ bot_username = self._status["telegram"].get("bot_username")+ logger.info(f"[StatusBroadcaster] Telegram auto-reconnected: @{bot_username}")+ else:+ logger.warning(f"[StatusBroadcaster] Telegram auto-reconnect failed: {result.get('error')}")- await self.broadcast({- "type": "telegram_status",- "data": self._status["telegram"],- })- except Exception as e:- logger.debug(f"[StatusBroadcaster] Could not refresh Telegram status: {e}")+ await self.broadcast({+ "type": "telegram_status",+ "data": self._status["telegram"],+ })+ span.set_attribute("connected", bool(self._status["telegram"]["connected"]))+ except Exception as e:+ span.record_exception(e)+ logger.debug(f"[StatusBroadcaster] Could not refresh Telegram status: {e}")async def _auto_reconnect_android_relay(self):"""Auto-reconnect to Android relay if there's a stored pairing session.Called on client connect to re-establish relay connection after server restart.The stored session contains relay URL, API key, and paired device info.+ Span emitted via OTel for cold-start benchmarking."""+ with tracer.start_as_current_span("broadcaster.refresh_android") as span:+ await self._auto_reconnect_android_relay_body(span)++ async def _auto_reconnect_android_relay_body(self, span):try:# Check if already connectedfrom services.android.manager import get_current_relay_client@@ -431,6 +467,7 @@ class StatusBroadcaster:"type": "android_status","data": self._status["android"],})+ span.set_attribute("path", "already_connected")return# Check for stored session@@ -439,6 +476,7 @@ class StatusBroadcaster:session = await database.get_android_relay_session()if not session:+ span.set_attribute("path", "no_session")logger.debug("[StatusBroadcaster] No stored Android relay session")return@@ -448,9 +486,11 @@ class StatusBroadcaster:session.get("device_name")if not relay_url or not api_key:+ span.set_attribute("path", "session_missing_creds")logger.debug("[StatusBroadcaster] Stored session missing relay URL or API key")return+ span.set_attribute("path", "auto_reconnect")logger.info("[StatusBroadcaster] Auto-reconnecting to Android relay...",relay_url=relay_url, device_id=device_id)@@ -477,12 +517,15 @@ class StatusBroadcaster:"type": "android_status","data": self._status["android"],})+ span.set_attribute("reconnect_ok", True)else:+ span.set_attribute("reconnect_ok", False)logger.warning(f"[StatusBroadcaster] Failed to reconnect Android relay: {error}")# Clear the stored session since reconnect failedawait database.clear_android_relay_session()except Exception as e:+ span.record_exception(e)logger.debug(f"[StatusBroadcaster] Could not auto-reconnect Android relay: {e}")async def update_whatsapp_status(diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.pyindex c8c47606..ae72cf29 100644--- a/server/services/temporal/worker.py+++ b/server/services/temporal/worker.py@@ -19,6 +19,7 @@ import asynciofrom typing import Optionalimport aiohttp+from opentelemetry import tracefrom temporalio.client import Clientfrom temporalio.runtime import LoggingConfig, Runtime, TelemetryConfigfrom temporalio.worker import Worker@@ -31,6 +32,7 @@ from .activities import ()logger = get_logger(__name__)+tracer = trace.get_tracer(__name__)def create_runtime() -> Runtime:@@ -81,46 +83,55 @@ class TemporalWorkerManager:return self._worker_task is not None and not self._worker_task.done()async def start(self) -> None:- """Start the Temporal worker in the background."""- if self.is_running:- logger.warning("Temporal worker already running")- return+ """Start the Temporal worker in the background.- # Create shared aiohttp session with connection pooling- self._session = await create_shared_session(self.pool_size)-- # Create activity instance with shared session- self._activities = NodeExecutionActivities(self._session)-- # MachinaWorkflow.run() schedules every node via the single- # ``"execute_node_activity"`` name (see services/temporal/workflow.py).- # The Wave 11.F per-plugin activities (``node.{type}.v{version}``)- # added a ~1.6s registration cost at every worker startup but were- # never scheduled by the orchestrator — pure dead code on the hot- # path. Removed. If per-node task-queue dispatch is revived, gate- # behind a Settings flag and wire it into the orchestrator first.- self._worker = Worker(- self.client,- task_queue=self.task_queue,- workflows=[MachinaWorkflow],- activities=[self._activities.execute_node_activity],- # Allow concurrent activity execution for parallel branches- max_concurrent_activities=self.pool_size,- max_concurrent_workflow_tasks=10,- )+ Span emitted via OTel for cold-start benchmarking — exposes the+ wall-clock cost of session creation, activity instantiation, and+ worker registration.+ """+ with tracer.start_as_current_span("temporal.worker_start") as span:+ if self.is_running:+ span.set_attribute("already_running", True)+ logger.warning("Temporal worker already running")+ return++ # Create shared aiohttp session with connection pooling+ self._session = await create_shared_session(self.pool_size)++ # Create activity instance with shared session+ self._activities = NodeExecutionActivities(self._session)++ # MachinaWorkflow.run() schedules every node via the single+ # ``"execute_node_activity"`` name (see services/temporal/workflow.py).+ # The Wave 11.F per-plugin activities (``node.{type}.v{version}``)+ # added a ~1.6s registration cost at every worker startup but were+ # never scheduled by the orchestrator — pure dead code on the hot+ # path. Removed. If per-node task-queue dispatch is revived, gate+ # behind a Settings flag and wire it into the orchestrator first.+ self._worker = Worker(+ self.client,+ task_queue=self.task_queue,+ workflows=[MachinaWorkflow],+ activities=[self._activities.execute_node_activity],+ # Allow concurrent activity execution for parallel branches+ max_concurrent_activities=self.pool_size,+ max_concurrent_workflow_tasks=10,+ )+ span.set_attribute("task_queue", self.task_queue)+ span.set_attribute("pool_size", self.pool_size)- print(f"[Temporal] Worker started (queue={self.task_queue}, pool={self.pool_size})", flush=True)- logger.info(- "Starting Temporal worker",- task_queue=self.task_queue,- pool_size=self.pool_size,- )+ print(f"[Temporal] Worker started (queue={self.task_queue}, pool={self.pool_size})", flush=True)+ logger.info(+ "Starting Temporal worker",+ task_queue=self.task_queue,+ pool_size=self.pool_size,+ )- # Run worker in background task- self._worker_task = asyncio.create_task(- self._run_worker(),- name="temporal-worker",- )+ # Run worker in background task+ self._worker_task = asyncio.create_task(+ self._run_worker(),+ name="temporal-worker",+ )async def _run_worker(self) -> None:"""Run the worker (background task)."""