Skip to content

Self-improving AI Employees for everyone

perf(observability): OpenTelemetry spans + Node perf_hooks for benchmarking

trohitg64a2f79fc3f8
diff --git a/scripts/dev.js b/scripts/dev.js
index 92835980..54c8254f 100644
--- a/scripts/dev.js
+++ b/scripts/dev.js
@@ -15,11 +15,17 @@ import {
killPort,
createLogger,
waitForTcpPort,
+ installPerfMeasureLogger,
} from './utils.js';
const START_TIME = Date.now();
const log = createLogger(START_TIME);
+// Install the perf observer once: every performance.measure entry from
+// this process (including the ones emitted by waitForTcpPort) is logged
+// as a structured "[perf] <name> <ms>" line. Standard Node lib only.
+installPerfMeasureLogger(log);
+
// Parse command line arguments
const args = process.argv.slice(2);
const isDaemonMode = args.includes('--daemon');
@@ -119,7 +125,10 @@ async function main() {
Promise.all(
readyTargets.map(async ({ name, port }) => {
- const ok = await waitForTcpPort(port);
+ // Pass `name` so the perf observer emits ``tcp.wait.<service>``
+ // entries instead of port-numbered ones. Timing comes from the
+ // observer (single source), not inline scaffolding here.
+ const ok = await waitForTcpPort(port, { name: name.toLowerCase() });
log(ok ? `${name} ready (port ${port})` : `${name} did not become ready (port ${port})`);
return ok;
})
diff --git a/scripts/start.js b/scripts/start.js
index bb82efdb..c414d4a0 100755
--- a/scripts/start.js
+++ b/scripts/start.js
@@ -16,11 +16,17 @@ import {
killPort,
createLogger,
waitForTcpPort,
+ installPerfMeasureLogger,
} from './utils.js';
const START_TIME = Date.now();
const log = createLogger(START_TIME);
+// One PerformanceObserver per process — every performance.measure entry
+// (including those from waitForTcpPort) is logged as a structured
+// "[perf] <name> <ms>" line. Standard Node lib only.
+installPerfMeasureLogger(log);
+
const args = process.argv.slice(2);
const isVerbose = args.includes('--verbose') || args.includes('-v');
const skipWhatsApp = args.includes('--skip-whatsapp');
@@ -127,7 +133,9 @@ async function main() {
function startReadyProbes() {
Promise.all(
readyTargets.map(async ({ name, port }) => {
- const ok = await waitForTcpPort(port);
+ // Pass `name` so the perf observer emits ``tcp.wait.<service>``
+ // entries instead of port-numbered ones.
+ const ok = await waitForTcpPort(port, { name: name.toLowerCase() });
log(ok ? `${name} ready (port ${port})` : `${name} did not become ready (port ${port})`);
return ok;
})
diff --git a/scripts/utils.js b/scripts/utils.js
index 346db814..19044be5 100644
--- a/scripts/utils.js
+++ b/scripts/utils.js
@@ -9,6 +9,7 @@
import { readFileSync, existsSync, copyFileSync } from 'fs';
import { Socket } from 'net';
import { resolve, dirname } from 'path';
+import { performance, PerformanceObserver } from 'node:perf_hooks';
import { fileURLToPath } from 'url';
import { execSync } from 'child_process';
@@ -81,17 +82,50 @@ export function probeTcpPort(port, host = '127.0.0.1', timeoutMs = 500) {
* ready to accept traffic — significantly more reliable and faster than
* grepping stdout for ready-message patterns, which trail actual readiness
* by hundreds of ms due to line-buffered stdout flush.
+ *
+ * Emits a Node ``performance.measure`` entry named ``tcp.wait.<name>``
+ * (or ``tcp.wait.<port>`` if no name) so callers that installed a
+ * ``PerformanceObserver`` (see ``installPerfMeasureLogger``) get
+ * structured benchmark output without per-callsite timing scaffolding.
*/
export async function waitForTcpPort(
port,
- { intervalMs = 250, maxWaitMs = 60000, host = '127.0.0.1' } = {}
+ { intervalMs = 250, maxWaitMs = 60000, host = '127.0.0.1', name } = {}
) {
+ const measureName = `tcp.wait.${name ?? port}`;
+ const startMark = `${measureName}.start.${performance.now()}`;
+ performance.mark(startMark);
const start = Date.now();
- while (Date.now() - start < maxWaitMs) {
- if (await probeTcpPort(port, host)) return true;
- await sleep(intervalMs);
+ try {
+ while (Date.now() - start < maxWaitMs) {
+ if (await probeTcpPort(port, host)) return true;
+ await sleep(intervalMs);
+ }
+ return false;
+ } finally {
+ performance.measure(measureName, startMark);
+ performance.clearMarks(startMark);
}
- return false;
+}
+
+/**
+ * Install a ``PerformanceObserver`` that logs every ``performance.measure``
+ * entry through the supplied logger. This is the Node stdlib equivalent
+ * of the OpenTelemetry "tracer + console exporter" pattern: instrumented
+ * code calls ``performance.mark/measure``; one observer at the script
+ * boundary turns those entries into structured log lines.
+ *
+ * Returns the observer so the caller can ``disconnect()`` if needed
+ * (rarely — observers are usually script-lifetime).
+ */
+export function installPerfMeasureLogger(log, { prefix = '[perf]' } = {}) {
+ const observer = new PerformanceObserver((items) => {
+ for (const entry of items.getEntries()) {
+ log(`${prefix} ${entry.name} ${Math.round(entry.duration)}ms`);
+ }
+ });
+ observer.observe({ entryTypes: ['measure'] });
+ return observer;
}
/**
diff --git a/server/core/tracing.py b/server/core/tracing.py
new file mode 100644
index 00000000..ea443019
--- /dev/null
+++ b/server/core/tracing.py
@@ -0,0 +1,60 @@
+"""OpenTelemetry tracing bootstrap.
+
+Single initialization point for the OTel TracerProvider. Modules that
+want to emit spans do::
+
+ from opentelemetry import trace
+ tracer = trace.get_tracer(__name__)
+
+ async def something():
+ with tracer.start_as_current_span("namespace.operation"):
+ ...
+
+Spans are exported via ``ConsoleSpanExporter`` so cold-start
+benchmarking is grep-able from the same terminal that runs the dev
+server::
+
+ pnpm run dev 2>&1 | grep '"name":'
+
+In production, swap ``ConsoleSpanExporter`` for an OTLP exporter
+pointed at a collector (Jaeger / Tempo / Honeycomb / etc.) — purely
+config, no code change at the call sites.
+
+This file owns the only place in the codebase where the OTel SDK is
+configured. Every other module uses ``trace.get_tracer(__name__)``
+which is the standard library API; no project-specific abstraction.
+"""
+
+from __future__ import annotations
+
+from opentelemetry import trace
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+ BatchSpanProcessor,
+ ConsoleSpanExporter,
+)
+
+
+_INITIALIZED = False
+
+
+def init_tracing(service_name: str = "machinaos-backend") -> None:
+ """Configure the global TracerProvider.
+
+ Idempotent — safe to call multiple times. Subsequent calls are
+ no-ops so tests / reloads do not accumulate processors.
+ """
+ global _INITIALIZED
+ if _INITIALIZED:
+ return
+
+ provider = TracerProvider(
+ resource=Resource.create({SERVICE_NAME: service_name})
+ )
+ # ConsoleSpanExporter writes one JSON document per span to stdout —
+ # ideal for dev/benchmark mode. Span fields include ``name``,
+ # ``start_time``, ``end_time``, ``attributes``, ``status``.
+ provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+ trace.set_tracer_provider(provider)
+ _INITIALIZED = True
diff --git a/server/main.py b/server/main.py
index 4d8efe20..29d7994e 100644
--- a/server/main.py
+++ b/server/main.py
@@ -38,13 +38,19 @@ from core.container import container
_startup_log("Importing settings + logging...")
from core.config import Settings
from core.logging import configure_logging, get_logger, setup_websocket_logging, shutdown_websocket_logging
+from core.tracing import init_tracing
_startup_log("Importing routers...")
from routers import workflow, database, maps, nodejs_compat, android, websocket, webhook, auth, twitter, google, credentials, schemas
_startup_log("All imports complete")
-# Initialize settings and logging
+# Initialize settings, logging, and tracing.
+# init_tracing() registers the OpenTelemetry TracerProvider with a
+# ConsoleSpanExporter so every span emitted via
+# ``trace.get_tracer(__name__).start_as_current_span(...)`` shows up in
+# stdout for cold-start benchmarking.
settings = Settings()
configure_logging(settings)
+init_tracing()
logger = get_logger(__name__)
# Suppress noisy loggers
diff --git a/server/pyproject.toml b/server/pyproject.toml
index 1ad91923..ea2fada7 100644
--- a/server/pyproject.toml
+++ b/server/pyproject.toml
@@ -27,6 +27,10 @@ dependencies = [
"structlog>=24.1.0",
"orjson>=3.9.0",
+ # Tracing / benchmarking (OpenTelemetry — industry standard)
+ "opentelemetry-api>=1.27.0",
+ "opentelemetry-sdk>=1.27.0",
+
# YAML parsing (for SKILL.md files)
"pyyaml>=6.0",
diff --git a/server/services/status_broadcaster.py b/server/services/status_broadcaster.py
index 58cb9054..fb316823 100644
--- a/server/services/status_broadcaster.py
+++ b/server/services/status_broadcaster.py
@@ -8,9 +8,12 @@ import asyncio
import orjson
from typing import Set, Dict, Any, Optional, List
from fastapi import WebSocket
+from opentelemetry import trace
+
from core.logging import get_logger
logger = get_logger(__name__)
+tracer = trace.get_tracer(__name__)
class StatusBroadcaster:
@@ -133,17 +136,22 @@ class StatusBroadcaster:
Each ``_refresh_*`` already swallows its own exceptions, so
TaskGroup never sees one. Kept inside try/except* defensively
in case a future refactor lets one escape.
+
+ Wrapped in an OTel span so the cycle's wall-clock duration is
+ emitted via the configured span exporter (ConsoleSpanExporter
+ in dev for benchmarking, OTLP in prod).
"""
- try:
- async with asyncio.TaskGroup() as tg:
- tg.create_task(self._refresh_whatsapp_status())
- tg.create_task(self._refresh_twitter_status())
- tg.create_task(self._refresh_google_status())
- tg.create_task(self._refresh_telegram_status())
- tg.create_task(self._auto_reconnect_android_relay())
- except* Exception as eg:
- for exc in eg.exceptions:
- logger.warning("[StatusBroadcaster] Service refresh task failed: %s", exc)
+ with tracer.start_as_current_span("broadcaster.refresh_all_services"):
+ try:
+ async with asyncio.TaskGroup() as tg:
+ tg.create_task(self._refresh_whatsapp_status())
+ tg.create_task(self._refresh_twitter_status())
+ tg.create_task(self._refresh_google_status())
+ tg.create_task(self._refresh_telegram_status())
+ tg.create_task(self._auto_reconnect_android_relay())
+ except* Exception as eg:
+ for exc in eg.exceptions:
+ logger.warning("[StatusBroadcaster] Service refresh task failed: %s", exc)
async def broadcast(self, message: Dict[str, Any]):
"""Broadcast a message to all connected clients using TaskGroup.
@@ -259,105 +267,117 @@ class StatusBroadcaster:
"""Fetch fresh WhatsApp status from Go service and update cache.
Called on client connect to ensure initial_status has accurate data.
- Silently fails if WhatsApp service is unavailable.
+ Silently fails if WhatsApp service is unavailable. Span emitted
+ via OTel for cold-start benchmarking.
"""
- try:
- from services.whatsapp_service import get_client
- import time
-
- client = await get_client()
- status_data = await client.call("status")
-
- self._status["whatsapp"] = {
- "connected": status_data.get("connected", False),
- "has_session": status_data.get("has_session", False),
- "running": status_data.get("running", False),
- "pairing": status_data.get("pairing", False),
- "device_id": status_data.get("device_id"),
- "qr": None,
- "timestamp": time.time()
- }
- logger.debug(f"[StatusBroadcaster] Refreshed WhatsApp status: connected={status_data.get('connected')}")
- await self.broadcast({
- "type": "whatsapp_status",
- "data": self._status["whatsapp"],
- })
- except Exception as e:
- # Don't fail client connection if WhatsApp service is down
- logger.debug(f"[StatusBroadcaster] Could not refresh WhatsApp status: {e}")
+ with tracer.start_as_current_span("broadcaster.refresh_whatsapp") as span:
+ try:
+ from services.whatsapp_service import get_client
+ import time
+
+ client = await get_client()
+ status_data = await client.call("status")
+
+ self._status["whatsapp"] = {
+ "connected": status_data.get("connected", False),
+ "has_session": status_data.get("has_session", False),
+ "running": status_data.get("running", False),
+ "pairing": status_data.get("pairing", False),
+ "device_id": status_data.get("device_id"),
+ "qr": None,
+ "timestamp": time.time()
+ }
+ logger.debug(f"[StatusBroadcaster] Refreshed WhatsApp status: connected={status_data.get('connected')}")
+ await self.broadcast({
+ "type": "whatsapp_status",
+ "data": self._status["whatsapp"],
+ })
+ span.set_attribute("connected", bool(status_data.get("connected", False)))
+ except Exception as e:
+ span.record_exception(e)
+ # Don't fail client connection if WhatsApp service is down
+ logger.debug(f"[StatusBroadcaster] Could not refresh WhatsApp status: {e}")
async def _refresh_twitter_status(self):
"""Fetch Twitter status from stored OAuth tokens in database.
Called on client connect to check if user is authenticated with Twitter.
Uses OAuth token system (matches REST endpoint and node handlers).
+ Span emitted via OTel for cold-start benchmarking.
"""
- try:
- from core.container import container
- auth_service = container.auth_service()
-
- # Get tokens from OAuth system (NOT api_key system)
- tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")
- if not tokens or not tokens.get("access_token"):
- self._status["twitter"] = {
- "connected": False,
- "username": None,
- "user_id": None,
- "name": None,
- "profile_image_url": None
- }
- else:
- # User info is stored in the OAuth token record
- email = tokens.get("email", "") # Stored as "@username"
- name = tokens.get("name", "")
- username = email.lstrip("@") if email.startswith("@") else email
-
- self._status["twitter"] = {
- "connected": True,
- "username": username or None,
- "user_id": None,
- "name": name or None,
- "profile_image_url": None
- }
- logger.debug(f"[StatusBroadcaster] Twitter status: connected as @{username}")
+ with tracer.start_as_current_span("broadcaster.refresh_twitter") as span:
+ try:
+ from core.container import container
+ auth_service = container.auth_service()
+
+ # Get tokens from OAuth system (NOT api_key system)
+ tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")
+ if not tokens or not tokens.get("access_token"):
+ self._status["twitter"] = {
+ "connected": False,
+ "username": None,
+ "user_id": None,
+ "name": None,
+ "profile_image_url": None
+ }
+ else:
+ # User info is stored in the OAuth token record
+ email = tokens.get("email", "") # Stored as "@username"
+ name = tokens.get("name", "")
+ username = email.lstrip("@") if email.startswith("@") else email
+
+ self._status["twitter"] = {
+ "connected": True,
+ "username": username or None,
+ "user_id": None,
+ "name": name or None,
+ "profile_image_url": None
+ }
+ logger.debug(f"[StatusBroadcaster] Twitter status: connected as @{username}")
- await self.broadcast({
- "type": "twitter_status",
- "data": self._status["twitter"],
- })
- except Exception as e:
- logger.debug(f"[StatusBroadcaster] Could not refresh Twitter status: {e}")
+ await self.broadcast({
+ "type": "twitter_status",
+ "data": self._status["twitter"],
+ })
+ span.set_attribute("connected", bool(self._status["twitter"]["connected"]))
+ except Exception as e:
+ span.record_exception(e)
+ logger.debug(f"[StatusBroadcaster] Could not refresh Twitter status: {e}")
async def _refresh_google_status(self):
"""Fetch Google Workspace status from stored OAuth tokens.
Called on client connect to check if user is authenticated with Google.
+ Span emitted via OTel for cold-start benchmarking.
"""
- try:
- from core.container import container
- auth_service = container.auth_service()
-
- tokens = await auth_service.get_oauth_tokens("google", customer_id="owner")
- if not tokens or not tokens.get("access_token"):
- self._status["google"] = {
- "connected": False,
- "email": None,
- "name": None,
- }
- else:
- self._status["google"] = {
- "connected": True,
- "email": tokens.get("email"),
- "name": tokens.get("name"),
- }
- logger.debug(f"[StatusBroadcaster] Google status: connected as {tokens.get('email')}")
+ with tracer.start_as_current_span("broadcaster.refresh_google") as span:
+ try:
+ from core.container import container
+ auth_service = container.auth_service()
+
+ tokens = await auth_service.get_oauth_tokens("google", customer_id="owner")
+ if not tokens or not tokens.get("access_token"):
+ self._status["google"] = {
+ "connected": False,
+ "email": None,
+ "name": None,
+ }
+ else:
+ self._status["google"] = {
+ "connected": True,
+ "email": tokens.get("email"),
+ "name": tokens.get("name"),
+ }
+ logger.debug(f"[StatusBroadcaster] Google status: connected as {tokens.get('email')}")
- await self.broadcast({
- "type": "google_status",
- "data": self._status["google"],
- })
- except Exception as e:
- logger.debug(f"[StatusBroadcaster] Could not refresh Google status: {e}")
+ await self.broadcast({
+ "type": "google_status",
+ "data": self._status["google"],
+ })
+ span.set_attribute("connected", bool(self._status["google"]["connected"]))
+ except Exception as e:
+ span.record_exception(e)
+ logger.debug(f"[StatusBroadcaster] Could not refresh Google status: {e}")
async def _refresh_telegram_status(self):
"""Refresh Telegram status from stored credentials and auto-reconnect.
@@ -366,50 +386,66 @@ class StatusBroadcaster:
inside TelegramService; this function just decides whether to
attempt an auto-reconnect and mirrors the resulting status into
the broadcaster cache.
- """
- try:
- from services.telegram_service import get_telegram_service
- service = get_telegram_service()
-
- def _snapshot(connected: bool, has_token: bool) -> Dict[str, Any]:
- s = service.get_status()
- return {
- "connected": connected,
- "bot_id": s.get("bot_id") if connected else None,
- "bot_username": s.get("bot_username") if connected else None,
- "bot_name": s.get("bot_name") if connected else None,
- "owner_chat_id": s.get("owner_chat_id") if connected else None,
- "has_stored_token": has_token,
- }
- if service.connected:
- self._status["telegram"] = _snapshot(True, True)
- elif not await service.has_stored_token():
- self._status["telegram"] = _snapshot(False, False)
- else:
- logger.info("[StatusBroadcaster] Auto-reconnecting Telegram bot...")
- result = await service.connect() # service reads token from DB
- ok = bool(result.get("success"))
- self._status["telegram"] = _snapshot(ok, True)
- if ok:
- bot_username = self._status["telegram"].get("bot_username")
- logger.info(f"[StatusBroadcaster] Telegram auto-reconnected: @{bot_username}")
+ Span emitted via OTel for cold-start benchmarking. This is the
+ canonical "is the broadcaster fast?" metric — Telegram's getMe
+ on cold DNS is the historical bottleneck (10s read_timeout).
+ """
+ with tracer.start_as_current_span("broadcaster.refresh_telegram") as span:
+ try:
+ from services.telegram_service import get_telegram_service
+ service = get_telegram_service()
+
+ def _snapshot(connected: bool, has_token: bool) -> Dict[str, Any]:
+ s = service.get_status()
+ return {
+ "connected": connected,
+ "bot_id": s.get("bot_id") if connected else None,
+ "bot_username": s.get("bot_username") if connected else None,
+ "bot_name": s.get("bot_name") if connected else None,
+ "owner_chat_id": s.get("owner_chat_id") if connected else None,
+ "has_stored_token": has_token,
+ }
+
+ if service.connected:
+ self._status["telegram"] = _snapshot(True, True)
+ span.set_attribute("path", "already_connected")
+ elif not await service.has_stored_token():
+ self._status["telegram"] = _snapshot(False, False)
+ span.set_attribute("path", "no_token")
else:
- logger.warning(f"[StatusBroadcaster] Telegram auto-reconnect failed: {result.get('error')}")
+ span.set_attribute("path", "auto_reconnect")
+ logger.info("[StatusBroadcaster] Auto-reconnecting Telegram bot...")
+ result = await service.connect() # service reads token from DB
+ ok = bool(result.get("success"))
+ self._status["telegram"] = _snapshot(ok, True)
+ span.set_attribute("reconnect_ok", ok)
+ if ok:
+ bot_username = self._status["telegram"].get("bot_username")
+ logger.info(f"[StatusBroadcaster] Telegram auto-reconnected: @{bot_username}")
+ else:
+ logger.warning(f"[StatusBroadcaster] Telegram auto-reconnect failed: {result.get('error')}")
- await self.broadcast({
- "type": "telegram_status",
- "data": self._status["telegram"],
- })
- except Exception as e:
- logger.debug(f"[StatusBroadcaster] Could not refresh Telegram status: {e}")
+ await self.broadcast({
+ "type": "telegram_status",
+ "data": self._status["telegram"],
+ })
+ span.set_attribute("connected", bool(self._status["telegram"]["connected"]))
+ except Exception as e:
+ span.record_exception(e)
+ logger.debug(f"[StatusBroadcaster] Could not refresh Telegram status: {e}")
async def _auto_reconnect_android_relay(self):
"""Auto-reconnect to Android relay if there's a stored pairing session.
Called on client connect to re-establish relay connection after server restart.
The stored session contains relay URL, API key, and paired device info.
+ Span emitted via OTel for cold-start benchmarking.
"""
+ with tracer.start_as_current_span("broadcaster.refresh_android") as span:
+ await self._auto_reconnect_android_relay_body(span)
+
+ async def _auto_reconnect_android_relay_body(self, span):
try:
# Check if already connected
from services.android.manager import get_current_relay_client
@@ -431,6 +467,7 @@ class StatusBroadcaster:
"type": "android_status",
"data": self._status["android"],
})
+ span.set_attribute("path", "already_connected")
return
# Check for stored session
@@ -439,6 +476,7 @@ class StatusBroadcaster:
session = await database.get_android_relay_session()
if not session:
+ span.set_attribute("path", "no_session")
logger.debug("[StatusBroadcaster] No stored Android relay session")
return
@@ -448,9 +486,11 @@ class StatusBroadcaster:
session.get("device_name")
if not relay_url or not api_key:
+ span.set_attribute("path", "session_missing_creds")
logger.debug("[StatusBroadcaster] Stored session missing relay URL or API key")
return
+ span.set_attribute("path", "auto_reconnect")
logger.info("[StatusBroadcaster] Auto-reconnecting to Android relay...",
relay_url=relay_url, device_id=device_id)
@@ -477,12 +517,15 @@ class StatusBroadcaster:
"type": "android_status",
"data": self._status["android"],
})
+ span.set_attribute("reconnect_ok", True)
else:
+ span.set_attribute("reconnect_ok", False)
logger.warning(f"[StatusBroadcaster] Failed to reconnect Android relay: {error}")
# Clear the stored session since reconnect failed
await database.clear_android_relay_session()
except Exception as e:
+ span.record_exception(e)
logger.debug(f"[StatusBroadcaster] Could not auto-reconnect Android relay: {e}")
async def update_whatsapp_status(
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
index c8c47606..ae72cf29 100644
--- a/server/services/temporal/worker.py
+++ b/server/services/temporal/worker.py
@@ -19,6 +19,7 @@ import asyncio
from typing import Optional
import aiohttp
+from opentelemetry import trace
from temporalio.client import Client
from temporalio.runtime import LoggingConfig, Runtime, TelemetryConfig
from temporalio.worker import Worker
@@ -31,6 +32,7 @@ from .activities import (
)
logger = get_logger(__name__)
+tracer = trace.get_tracer(__name__)
def create_runtime() -> Runtime:
@@ -81,46 +83,55 @@ class TemporalWorkerManager:
return self._worker_task is not None and not self._worker_task.done()
async def start(self) -> None:
- """Start the Temporal worker in the background."""
- if self.is_running:
- logger.warning("Temporal worker already running")
- return
+ """Start the Temporal worker in the background.
- # Create shared aiohttp session with connection pooling
- self._session = await create_shared_session(self.pool_size)
-
- # Create activity instance with shared session
- self._activities = NodeExecutionActivities(self._session)
-
- # MachinaWorkflow.run() schedules every node via the single
- # ``"execute_node_activity"`` name (see services/temporal/workflow.py).
- # The Wave 11.F per-plugin activities (``node.{type}.v{version}``)
- # added a ~1.6s registration cost at every worker startup but were
- # never scheduled by the orchestrator — pure dead code on the hot
- # path. Removed. If per-node task-queue dispatch is revived, gate
- # behind a Settings flag and wire it into the orchestrator first.
- self._worker = Worker(
- self.client,
- task_queue=self.task_queue,
- workflows=[MachinaWorkflow],
- activities=[self._activities.execute_node_activity],
- # Allow concurrent activity execution for parallel branches
- max_concurrent_activities=self.pool_size,
- max_concurrent_workflow_tasks=10,
- )
+ Span emitted via OTel for cold-start benchmarking — exposes the
+ wall-clock cost of session creation, activity instantiation, and
+ worker registration.
+ """
+ with tracer.start_as_current_span("temporal.worker_start") as span:
+ if self.is_running:
+ span.set_attribute("already_running", True)
+ logger.warning("Temporal worker already running")
+ return
+
+ # Create shared aiohttp session with connection pooling
+ self._session = await create_shared_session(self.pool_size)
+
+ # Create activity instance with shared session
+ self._activities = NodeExecutionActivities(self._session)
+
+ # MachinaWorkflow.run() schedules every node via the single
+ # ``"execute_node_activity"`` name (see services/temporal/workflow.py).
+ # The Wave 11.F per-plugin activities (``node.{type}.v{version}``)
+ # added a ~1.6s registration cost at every worker startup but were
+ # never scheduled by the orchestrator — pure dead code on the hot
+ # path. Removed. If per-node task-queue dispatch is revived, gate
+ # behind a Settings flag and wire it into the orchestrator first.
+ self._worker = Worker(
+ self.client,
+ task_queue=self.task_queue,
+ workflows=[MachinaWorkflow],
+ activities=[self._activities.execute_node_activity],
+ # Allow concurrent activity execution for parallel branches
+ max_concurrent_activities=self.pool_size,
+ max_concurrent_workflow_tasks=10,
+ )
+ span.set_attribute("task_queue", self.task_queue)
+ span.set_attribute("pool_size", self.pool_size)
- print(f"[Temporal] Worker started (queue={self.task_queue}, pool={self.pool_size})", flush=True)
- logger.info(
- "Starting Temporal worker",
- task_queue=self.task_queue,
- pool_size=self.pool_size,
- )
+ print(f"[Temporal] Worker started (queue={self.task_queue}, pool={self.pool_size})", flush=True)
+ logger.info(
+ "Starting Temporal worker",
+ task_queue=self.task_queue,
+ pool_size=self.pool_size,
+ )
- # Run worker in background task
- self._worker_task = asyncio.create_task(
- self._run_worker(),
- name="temporal-worker",
- )
+ # Run worker in background task
+ self._worker_task = asyncio.create_task(
+ self._run_worker(),
+ name="temporal-worker",
+ )
async def _run_worker(self) -> None:
"""Run the worker (background task)."""