Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Temporal worker for distributed node execution.
Uses class-based activities with shared aiohttp session for proper
connection pooling across concurrent activity executions.
The worker polls the task queue and executes:
- MachinaWorkflow: Orchestrates the graph, schedules node activities
- NodeExecutionActivities: Executes individual nodes with shared session
Multiple workers can be started on different machines for horizontal scaling.
Each node activity can execute on any available worker in the cluster.
References:
- https://docs.temporal.io/develop/python/python-sdk-sync-vs-async
- https://docs.temporal.io/develop/worker-performance
"""
import asyncio
from datetime import timedelta
from typing import Optional
import aiohttp
from opentelemetry import trace
from temporalio.contrib.opentelemetry import TracingInterceptor
from temporalio.client import Client
from temporalio.runtime import LoggingConfig, Runtime, TelemetryConfig
from temporalio.worker import PollerBehaviorAutoscaling, Worker
from core.logging import get_logger
from ._interceptors import ObservabilityWorkerInterceptor
from .workflow import MachinaWorkflow
from .trigger_listener_workflow import TriggerListenerWorkflow
from .polling_trigger_workflow import PollingTriggerWorkflow
from .workflow_control_workflow import WorkflowControlWorkflow
from .plugin_registry import temporal_plugins
from .activities import (
NodeExecutionActivities,
create_shared_session,
)
from .execution_control_activities import ExecutionControlActivities
from .workflow_registry import framework_workflows as _framework_workflows
logger = get_logger(__name__)
tracer = trace.get_tracer(__name__)
def _graceful_shutdown_timeout() -> timedelta:
"""Wave 12 A3: SIGTERM grace window for Temporal workers.
Read from ``Settings.temporal_graceful_shutdown_seconds`` at the
moment each Worker is instantiated (rather than module load) so
test fixtures + env reloads pick up overrides.
"""
from core.config import Settings
return timedelta(seconds=Settings().temporal_graceful_shutdown_seconds)
def _worker_identity(queue: str) -> str:
"""Wave 17.4: explicit worker identity for Temporal Web UI ops.
Default SDK identity is ``<pid>@<hostname>`` which makes the
Workers tab unreadable once the per-queue pool runs 9 workers on
one host. ``opencompany-<queue>-<deployment_mode>`` names the role and
the topology at a glance.
"""
from core.config import Settings
return f"opencompany-{queue}-{Settings().deployment_mode}"
def _sticky_cache_size() -> int:
"""Wave 18.2: sticky workflow cache budget by deployment mode.
Cached workflow executions skip Event-History replay on the next
workflow task (docs.temporal.io/develop/worker-performance). Cache
entries hold the workflow's Python state in memory, so the budget
tracks host RAM: laptops get a small cache, always-on boxes a big
one. Evictions surface as ``sticky_cache_evictions`` + replay cost,
not errors — sizing is a latency knob, not a correctness one.
"""
from core.config import Settings
return {"local": 50, "cloud": 500, "self_hosted": 100}[Settings().deployment_mode]
def create_runtime() -> Runtime:
"""Create a Temporal runtime with worker heartbeating disabled.
Disables the runtime-level worker heartbeating feature to avoid
the warning on older Temporal server versions that don't support it.
"""
from services.temporal.client import CORE_LOG_FILTER
return Runtime(
telemetry=TelemetryConfig(
logging=LoggingConfig(filter=CORE_LOG_FILTER),
),
worker_heartbeat_interval=None, # Disable runtime heartbeating
)
class TemporalWorkerManager:
"""Manages the Temporal worker lifecycle with shared resources.
Creates a shared aiohttp.ClientSession that is passed to the activity
class, following Temporal's recommended dependency injection pattern.
"""
def __init__(
self,
client: Client,
task_queue: str = "machina-tasks",
pool_size: int = 100,
):
"""Initialize the worker manager.
Args:
client: Connected Temporal client
task_queue: Task queue name to poll
pool_size: Connection pool size for aiohttp session
"""
self.client = client
self.task_queue = task_queue
self.pool_size = pool_size
self._worker: Optional[Worker] = None
self._worker_task: Optional[asyncio.Task] = None
self._session: Optional[aiohttp.ClientSession] = None
self._activities: Optional[NodeExecutionActivities] = None
self._browser_pool = None
self._pi_driver_pool = None
@property
def is_running(self) -> bool:
"""Check if the worker is running."""
return self._worker_task is not None and not self._worker_task.done()
async def start(self) -> None:
"""Start the Temporal worker in the background.
Span emitted via OTel for cold-start benchmarking — exposes the
wall-clock cost of session creation, activity instantiation, and
worker registration.
"""
with tracer.start_as_current_span("temporal.worker_start") as span:
if self.is_running:
span.set_attribute("already_running", True)
logger.warning("Temporal worker already running")
return
# Create shared aiohttp session with connection pooling
self._session = await create_shared_session(self.pool_size)
# Create activity instance with shared session
self._activities = NodeExecutionActivities(self._session)
try:
self._worker = await self._build_worker()
# Maintenance drivers wait on framework/model/tool work. They
# must have independent slots even in monolithic deployments.
self._pi_driver_pool = PiDriverWorkerPool(self.client)
await self._pi_driver_pool.start()
except BaseException:
await self.stop()
raise
span.set_attribute("task_queue", self.task_queue)
span.set_attribute("pool_size", self.pool_size)
logger.info(
"Starting Temporal worker",
task_queue=self.task_queue,
pool_size=self.pool_size,
)
# Run worker in background task
self._worker_task = asyncio.create_task(
self._run_worker(),
name="temporal-worker",
)
async def _build_worker(self) -> Worker:
"""Construct a fresh, ready-to-run Worker instance.
Called from :meth:`start` AND from every crash-restart attempt in
:meth:`_run_worker` — the SDK ``Worker`` is single-use (a second
``run()`` on the same instance raises ``RuntimeError("Already
started")``), so restarting must always rebuild.
"""
# F4.A: register per-type activities alongside the legacy
# `execute_node_activity` dispatcher. The orchestrator at
# workflow.py:_resolve_activity picks one of the two paths per node
# based on the temporal_per_type_dispatch flag; both must be served
# by the worker. Per-type activities register WITHOUT a task_queue
# filter here (cls.task_queue is the *declared* preference; this
# framework worker polls self.task_queue regardless). Per-queue
# filtering lives in TemporalWorkerPool (below in this file),
# which is default-on (Wave 16.4) and runs one activity-only
# worker per declared queue. Registration cost: ~1.6s startup;
# zero runtime cost when the flag is off (orchestrator routes to
# execute_node_activity, per-type entries sit idle).
#
# F4.B: register AgentWorkflow + its agent activities
# (execute_llm_step / persist_turn / compact_context / ...). The
# orchestrator schedules AgentWorkflow as a child workflow
# for the 15 migrating agent types when
# ``temporal_agent_workflow_enabled`` is on.
from services.temporal.plugin_activities import (
collect_plugin_activities,
collect_polling_activities,
)
from services.temporal.agent_activities import collect_agent_activities
from services.temporal.pi_activities import collect_pi_activities, PiControlActivities
from services.temporal.pi_inline_delegation import collect_pi_inline_activities
from services.temporal.workspace_task_activities import reset_workspace_task_runtime
from services.chat.activities import CHAT_RUN_ACTIVITIES
from services.approvals.activities import APPROVAL_ACTIVITIES
from services.temporal.activities import (
broadcast_trigger_status_activity,
evaluate_trigger_filter_activity,
load_persisted_workflow_graph_activity,
pause_workflow_on_failure_activity,
record_run_completion_activity,
store_node_output_activity,
)
per_type = _generic_plugin_activities()
from services.temporal.browser_task_activities import HISTORY_ACTIVITIES, BROWSER_OWNER_ACTIVITIES
from core.config import Settings
local_browser_activities = [] if getattr(Settings(), "distributed_mode", False) is True else BROWSER_OWNER_ACTIVITIES
agent_activities = [*collect_agent_activities(), *collect_pi_activities(), *collect_pi_inline_activities(), *PiControlActivities(self.client).activities()]
polling_activities = collect_polling_activities()
# Plugin-owned workflow classes (e.g. cron's
# CronTriggerWorkflow) self-register a SimplePlugin via
# services.temporal.plugin_registry.register_temporal_plugin
# from their plugin __init__.py. The Temporal SDK's plugin
# chain merges each registered plugin's workflows / activities
# / interceptors into the effective worker configuration —
# the framework worker stays plugin-agnostic.
plugin_list = temporal_plugins()
# The Worker() constructor derives its default build id by
# MD5-hashing the bytecode of every module in sys.modules
# (disk reads included) — a synchronous ~3s stall at our
# module count (152 plugins + LLM SDK stack) that freezes
# the event loop and starves concurrent boot work (e.g.
# broadcaster refreshes). The result is memoized in an SDK
# module global, so one off-loop call here makes this
# Worker AND every TemporalWorkerPool worker construct
# cheaply. load_default_build_id is not re-exported from
# temporalio.worker (private module in 1.30); if an SDK
# upgrade moves it, skip the pre-warm and pay the
# synchronous cost again rather than fail startup.
try:
from temporalio.worker._worker import load_default_build_id
except ImportError:
logger.warning(
"temporalio private API moved: load_default_build_id "
"unavailable, skipping off-loop build-id pre-warm"
)
else:
await asyncio.to_thread(load_default_build_id)
worker = Worker(
self.client,
task_queue=self.task_queue,
plugins=plugin_list,
workflows=_framework_workflows(),
activities=[
self._activities.execute_node_activity,
*ExecutionControlActivities(self.client).activities(),
broadcast_trigger_status_activity,
evaluate_trigger_filter_activity,
load_persisted_workflow_graph_activity,
pause_workflow_on_failure_activity,
record_run_completion_activity,
reset_workspace_task_runtime,
*HISTORY_ACTIVITIES,
*local_browser_activities,
*CHAT_RUN_ACTIVITIES,
*APPROVAL_ACTIVITIES,
store_node_output_activity,
*per_type,
*agent_activities,
*polling_activities,
],
# Allow concurrent activity execution for parallel branches
max_concurrent_activities=self.pool_size,
max_concurrent_workflow_tasks=10,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
identity=_worker_identity(self.task_queue),
# ``TracingInterceptor`` is registered once on the shared client;
# the SDK prepends client interceptors to every worker, so
# repeating it here double-instruments each Signal / Query /
# Update / Activity and doubles the emitted spans.
interceptors=[ObservabilityWorkerInterceptor()],
# Wave 18.2: sticky cache sized by deployment mode so
# cached workflows skip Event-History replay without
# blowing laptop RAM.
max_cached_workflows=_sticky_cache_size(),
# Wave 18.3: autoscaling pollers replace fixed counts —
# scale between min/max on demand (SDK-recommended over
# manual poller tuning). Pollers stay well below the
# executor slot counts per the worker-performance docs.
activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=2, minimum=1, maximum=10),
workflow_task_poller_behavior=PollerBehaviorAutoscaling(initial=2, minimum=1, maximum=20),
)
logger.info(
"Registered Temporal activities",
legacy=1,
per_type=len(per_type),
agent=len(agent_activities),
task_queue=self.task_queue,
)
return worker
async def _run_worker(self) -> None:
"""Run the worker (background task), self-restarting on crash.
The Temporal worker shuts down on a poll failure rather than
auto-retrying (per the Python ``Worker`` docs), and this task is
detached — the startup retry loop in ``main.py`` has already
returned by the time it runs. So a transient crash (e.g. the
server briefly unavailable mid-poll) would otherwise leave the
worker permanently dead until a process restart.
Every restart attempt REBUILDS the worker via
:meth:`_build_worker`: the SDK ``Worker`` is single-use, so
re-running the same instance raises ``RuntimeError("Already
started")`` on the first retry and every one after it — the old
loop spun forever without ever polling again. Cancellation (from
``stop()``) always wins so shutdown is never delayed by a restart.
Backoff knobs are env-driven (canonical defaults in .env.template).
"""
from core.config import Settings
_s = Settings()
backoff = _s.temporal_worker_restart_backoff_seconds
backoff_max = _s.temporal_worker_restart_backoff_max_seconds
while True:
try:
await self._worker.run()
return # clean shutdown only
except asyncio.CancelledError:
logger.info("Temporal worker cancelled")
raise
except Exception as e:
logger.error(f"Temporal worker crashed; restarting in {backoff:.1f}s: {e}")
try:
await asyncio.sleep(backoff)
except asyncio.CancelledError:
raise # cancellation during backoff still wins
backoff = min(backoff * 2, backoff_max)
try:
self._worker = await self._build_worker()
except Exception as build_exc: # noqa: BLE001 — retried next loop
# Next iteration's run() on the stale single-use instance
# fails fast and re-enters this backoff, so rebuild
# failures are retried rather than fatal.
logger.error(f"Temporal worker rebuild failed; will retry: {build_exc}")
async def stop(self) -> None:
"""Stop the Temporal worker and cleanup resources."""
if self._pi_driver_pool is not None:
await self._pi_driver_pool.stop()
self._pi_driver_pool = None
if self._browser_pool is not None:
await self._browser_pool.stop()
self._browser_pool = None
logger.info("Stopping Temporal worker")
if self._worker_task:
self._worker_task.cancel()
try:
await self._worker_task
except asyncio.CancelledError:
pass
except Exception as exc:
logger.warning("Worker task already failed during shutdown: %s", type(exc).__name__)
self._worker_task = None
# Close shared session
if self._session and not self._session.closed:
await self._session.close()
self._worker = None
self._session = None
self._activities = None
logger.info("Temporal worker stopped")
class TemporalWorkerPool:
"""Wave 11.F: multi-queue worker pool.
Runs one :class:`Worker` per declared plugin ``task_queue``. Each
worker only registers activities whose plugin opts in to that
queue, so specialised workloads can be scaled independently.
Env overrides for per-pool concurrency: ``TEMPORAL_<QUEUE>_CONCURRENCY``
(e.g. ``TEMPORAL_AI_HEAVY_CONCURRENCY=4``).
Usage::
pool = TemporalWorkerPool(client, queues=["rest-api", "ai-heavy"])
await pool.start()
try:
...
finally:
await pool.stop()
"""
# Default concurrency per queue — override via env.
DEFAULT_CONCURRENCY: dict[str, int] = {
"machina-default": 20,
"rest-api": 50,
"ai-heavy": 4,
"code-exec": 10,
"triggers-poll": 100,
"triggers-event": 100,
"android": 10,
"browser": 4,
"messaging": 20,
}
# Wave 18.1: worker-level activities/second ceilings — protect
# external API quotas from runaway fan-out (Anthropic/OpenAI tier
# limits on ai-heavy; provider send limits on messaging). None =
# unthrottled (local subprocess work has no external quota).
# Override via TEMPORAL_<QUEUE>_RATE_LIMIT (float, per second).
DEFAULT_RATE_LIMIT: dict[str, Optional[float]] = {
"machina-default": None,
"rest-api": 100.0,
"ai-heavy": 60.0,
"code-exec": None,
"triggers-poll": None,
"triggers-event": None,
"android": None,
"browser": 10.0,
"messaging": 20.0,
}
def __init__(
self,
client: Client,
*,
queues: Optional[list[str]] = None,
default_pool_size: int = 20,
):
self.client = client
self.default_pool_size = default_pool_size
# Lazy import to avoid circular at module load time.
from services.temporal.plugin_activities import distinct_task_queues
self.queues = queues or distinct_task_queues()
self._workers: list[Worker] = []
self._tasks: list[asyncio.Task] = []
def _concurrency_for(self, queue: str) -> int:
"""Slots for a queue: explicit env override > mode-scaled default.
Wave 17.5: ``deployment_mode == "local"`` halves the per-queue
defaults (floor 1) — a dev laptop sharing 4-8 cores with an IDE
+ browser shouldn't run cloud-sized activity fan-out. Explicit
``TEMPORAL_<QUEUE>_CONCURRENCY`` env vars always win.
"""
import os
env_key = f"TEMPORAL_{queue.upper().replace('-', '_')}_CONCURRENCY"
raw = os.environ.get(env_key)
if raw and raw.isdigit():
return int(raw)
base = self.DEFAULT_CONCURRENCY.get(queue, self.default_pool_size)
from core.config import Settings
if Settings().deployment_mode == "local":
return max(1, base // 2)
return base
def _rate_limit_for(self, queue: str) -> Optional[float]:
"""Wave 18.1: activities/second ceiling for a queue's worker.
Explicit ``TEMPORAL_<QUEUE>_RATE_LIMIT`` env override (float)
wins; falls back to ``DEFAULT_RATE_LIMIT``; None = unthrottled.
"""
import os
env_key = f"TEMPORAL_{queue.upper().replace('-', '_')}_RATE_LIMIT"
raw = os.environ.get(env_key)
if raw:
try:
return float(raw)
except ValueError:
logger.warning(f"[Pool] Ignoring non-numeric {env_key}={raw!r}")
return self.DEFAULT_RATE_LIMIT.get(queue)
# Wave 18.4: queues whose workloads are too unpredictable for fixed
# slot counts. LLM turns range 2s-5min; browser sessions idle then
# spike on page render. The resource-based supplier targets 80%
# host CPU + memory instead — per the worker-performance docs it
# "can provide reasonable performance without the need for
# profiling". Other queues keep fixed sizing (predictable work).
RESOURCE_TUNED_QUEUES: frozenset = frozenset({"ai-heavy", "browser"})
def _tuner_for(self, queue: str, concurrency: int):
"""Return a WorkerTuner for resource-tuned queues, else None.
The Worker rejects ``tuner`` + ``max_concurrent_activities``
together, so the caller passes exactly one of the two. The
fixed workflow/local/nexus suppliers only exist to satisfy
``create_composite`` — pool workers host no workflows.
"""
if queue not in self.RESOURCE_TUNED_QUEUES:
return None
from temporalio.worker import (
FixedSizeSlotSupplier,
ResourceBasedSlotConfig,
ResourceBasedSlotSupplier,
ResourceBasedTunerConfig,
WorkerTuner,
)
tuner_config = ResourceBasedTunerConfig(target_memory_usage=0.8, target_cpu_usage=0.8)
return WorkerTuner.create_composite(
workflow_supplier=FixedSizeSlotSupplier(10),
activity_supplier=ResourceBasedSlotSupplier(
# minimum keeps the queue live under load spikes elsewhere
# on the host; maximum reuses the per-queue concurrency
# (env-overridable) as the hard ceiling.
ResourceBasedSlotConfig(minimum_slots=1, maximum_slots=max(concurrency, 2)),
tuner_config,
),
local_activity_supplier=FixedSizeSlotSupplier(10),
nexus_supplier=FixedSizeSlotSupplier(10),
)
@property
def is_running(self) -> bool:
return any(t is not None and not t.done() for t in self._tasks)
def _build_queue_worker(self, queue: str) -> Optional[Worker]:
"""Construct a fresh Worker for one queue.
Returns ``None`` when the queue has no registered activities.
Called from :meth:`start` AND from every crash-restart attempt in
:meth:`_run_queue_worker` — the SDK ``Worker`` is single-use, so
restarting must always rebuild.
"""
from services.temporal.plugin_activities import (
collect_plugin_activities,
)
activities = collect_plugin_activities(task_queue=queue)
# Automatic Coding Agent capabilities keep their plugins' queues and
# policies, but execute through a framework leaf rather than a canvas
# node. Register that leaf on the same queues when pool routing is on.
from services.coding_workspace_tools import WORKSPACE_TOOL_TYPES
from services.node_registry import get_node_class
if any((cls := get_node_class(node_type)) is not None and cls.task_queue == queue
for node_type in WORKSPACE_TOOL_TYPES):
from services.temporal.agent_activities import coding_workspace_tool
activities = [*activities, coding_workspace_tool]
from core.config import Settings
if getattr(Settings(), "distributed_mode", False) is True and queue == "browser":
return None
if not activities:
return None
concurrency = self._concurrency_for(queue)
rate_limit = self._rate_limit_for(queue)
tuner = self._tuner_for(queue, concurrency)
worker_kwargs: dict = dict(
task_queue=queue,
activities=activities,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
identity=_worker_identity(queue),
# ``TracingInterceptor`` is registered once on the shared client;
# the SDK prepends client interceptors to every worker, so
# repeating it here double-instruments each Signal / Query /
# Update / Activity and doubles the emitted spans.
interceptors=[ObservabilityWorkerInterceptor()],
# Wave 18.1: per-queue activities/second ceiling.
max_activities_per_second=rate_limit,
# Wave 18.3: activity-only workers need a small
# autoscaling poller budget — specialised queues see
# lower task volume than the manager's default queue.
activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=1, minimum=1, maximum=5),
)
# Wave 18.4: the Worker rejects `tuner` alongside ANY of
# `max_concurrent_workflow_tasks` / `max_concurrent_activities` /
# `max_concurrent_local_activities` / `max_concurrent_nexus_tasks`
# — the SDK enforces "size via tuner OR via max_concurrent_*,
# never both" (temporalio.worker.Worker.__init__). Resource-tuned
# queues get the tuner (which already carries a
# FixedSizeSlotSupplier(10) for workflow slots via
# `_tuner_for`); the rest keep fixed sizing.
if tuner is not None:
worker_kwargs["tuner"] = tuner
else:
worker_kwargs["max_concurrent_activities"] = concurrency
worker_kwargs["max_concurrent_workflow_tasks"] = 10
worker = Worker(self.client, **worker_kwargs)
logger.info(
f"[Pool] Built worker queue={queue!r} "
f"activities={len(activities)} "
f"slots={'resource-tuned<=' + str(max(concurrency, 2)) if tuner else concurrency} "
f"rate_limit={rate_limit if rate_limit is not None else 'unthrottled'}"
)
return worker
async def _run_queue_worker(self, queue: str, worker: Worker) -> None:
"""Run one queue's worker, rebuilding + restarting on crash.
Previously each queue worker ran as a bare ``worker.run()`` task:
a crash silently killed the ONLY worker for that queue and every
activity routed there pended forever (its workflows stalled)
until a backend restart. Same backoff + rebuild contract as
:meth:`TemporalWorkerManager._run_worker`; cancellation (from
:meth:`stop`) always wins.
"""
from core.config import Settings
_s = Settings()
backoff = _s.temporal_worker_restart_backoff_seconds
backoff_max = _s.temporal_worker_restart_backoff_max_seconds
current = worker
while True:
try:
await current.run()
return # clean shutdown only
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 — supervised restart
logger.error(f"[Pool] Worker for queue {queue!r} crashed; restarting in {backoff:.1f}s: {exc}")
try:
await asyncio.sleep(backoff)
except asyncio.CancelledError:
raise # cancellation during backoff still wins
backoff = min(backoff * 2, backoff_max)
rebuilt = self._build_queue_worker(queue)
if rebuilt is None:
logger.error(f"[Pool] Queue {queue!r} has no activities on rebuild; worker not restarted")
return
current = rebuilt
async def start(self) -> None:
for queue in self.queues:
worker = self._build_queue_worker(queue)
if worker is None:
logger.info(f"[Pool] Skipping empty queue {queue!r}")
continue
task = asyncio.create_task(
self._run_queue_worker(queue, worker),
name=f"worker-{queue}",
)
# Initial instances only — after a crash-restart the live
# instance is owned by the supervising task.
self._workers.append(worker)
self._tasks.append(task)
async def stop(self) -> None:
for task in self._tasks:
task.cancel()
for task in self._tasks:
try:
await task
except (asyncio.CancelledError, Exception):
pass
self._workers.clear()
self._tasks.clear()
logger.info("[Pool] All workers stopped")
def _generic_plugin_activities() -> list:
"""Generic workers cannot operate process-local browsers in a cluster."""
from core.config import Settings
from services.node_registry import registered_node_classes
from services.temporal.plugin_activities import collect_plugin_activities
if getattr(Settings(), "distributed_mode", False) is not True:
return collect_plugin_activities()
return collect_plugin_activities(include_types=[kind for kind in registered_node_classes() if kind != "browser"])
class BrowserOwnerWorkerPool(TemporalWorkerPool):
"""The backend alone polls its physical owner queue, independently of pool flags."""
def __init__(self, client: Client, owner_id: str):
from services.browser_owners import owner_queue
super().__init__(client, queues=[owner_queue(owner_id)], default_pool_size=4)
def _build_queue_worker(self, queue: str) -> Worker:
from services.temporal.plugin_activities import collect_plugin_activities
from services.temporal.browser_task_activities import BROWSER_OWNER_ACTIVITIES
# Filtering by this physical queue would discard the Browser class:
# its declared logical queue remains "browser".
return Worker(
self.client, task_queue=queue,
activities=[*collect_plugin_activities(include_types=["browser"]), *BROWSER_OWNER_ACTIVITIES],
max_concurrent_activities=4,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
identity=_worker_identity(queue),
interceptors=[ObservabilityWorkerInterceptor()],
)
class PiDriverWorkerPool(TemporalWorkerPool):
"""Always-on, separately bounded capacity for Pi maintenance drivers.
Parent drivers park while delegated children run, so a single driver
slot supports nested Coding Agent delegation without a capacity cycle.
The normal plugin worker-pool flag never controls this queue.
"""
QUEUE = "coding-pi-driver"
def __init__(self, client: Client):
super().__init__(client, queues=[self.QUEUE], default_pool_size=2)
def _concurrency_for(self, queue: str) -> int:
import os
from core.config import Settings
key = "TEMPORAL_CODING_PI_DRIVER_CONCURRENCY"
raw = os.environ.get(key)
if raw is not None:
try:
slots = int(raw)
except ValueError as exc:
raise ValueError(f"{key} must be a positive integer") from exc
if slots <= 0:
raise ValueError(f"{key} must be a positive integer")
return slots
return 1 if Settings().deployment_mode == "local" else 2
def _build_queue_worker(self, queue: str) -> Worker:
from services.temporal.pi_activities import PiDriverActivities
return Worker(
self.client,
task_queue=queue,
activities=PiDriverActivities(self.client).activities(),
max_concurrent_activities=self._concurrency_for(queue),
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
identity=_worker_identity(queue),
interceptors=[ObservabilityWorkerInterceptor()],
activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=1, minimum=1, maximum=2),
)
async def create_pi_driver_worker(client: Client) -> Worker:
"""Create the separate driver worker for tests/custom framework setups."""
pool = PiDriverWorkerPool(client)
return pool._build_queue_worker(pool.QUEUE)
async def run_standalone_worker(
server_address: str | None = None,
namespace: str = "default",
task_queue: str = "machina-tasks",
pool_size: int = 100,
) -> None:
"""Run the Temporal worker as a standalone process.
This can be used for running workers separately from the main server,
enabling horizontal scaling across multiple machines.
Example:
# Start multiple workers for horizontal scaling
python -m services.temporal.worker
Args:
server_address: Temporal server address. Defaults to
``TEMPORAL_SERVER_ADDRESS`` (canonical value in ``.env.template``
via ``core.env_defaults``) so a bare
``python -m services.temporal.worker`` dials the same server
as the backend.
namespace: Temporal namespace
task_queue: Task queue to poll
pool_size: Connection pool size
"""
if server_address is None:
from core.env_defaults import env_value
server_address = env_value("TEMPORAL_SERVER_ADDRESS")
logger.info(
"Starting standalone Temporal worker",
server_address=server_address,
namespace=namespace,
task_queue=task_queue,
pool_size=pool_size,
)
# Embedded workers get this from main.py's lifespan; standalone
# workers must load the model registry themselves or max_tokens /
# context_length lookups degrade to hard fallbacks (4096).
from services.model_registry import get_model_registry
get_model_registry().startup()
# Use custom runtime with heartbeating disabled to avoid warning on older servers
runtime = create_runtime()
# Connect with retries (server may still be starting)
client = None
for attempt in range(1, 6):
try:
logger.info(f"Connecting to Temporal server (attempt {attempt}/5)")
client = await Client.connect(
server_address,
namespace=namespace,
runtime=runtime,
interceptors=[TracingInterceptor()],
)
logger.info("Connected to Temporal server")
break
except Exception as e:
logger.warning(f"Temporal connection attempt {attempt}/5 failed: {e}")
if attempt < 5:
await asyncio.sleep(3.0)
if client is None:
logger.error(f"Could not connect to Temporal server at {server_address} after 5 attempts")
return
from core.config import Settings
# Pi checkpoint Activities always use the shared database, including a
# standalone local worker. Initialize its tables before either queue polls.
from core.container import container
if getattr(Settings(), "distributed_mode", False) is True:
import nodes # noqa: F401 -- populate the native plugin registry
await container.database().startup()
# Context pressure relief runs here without the backend's lifespan. Its
# Activity resolves the shared compactor, which needs the same native
# AI service and provider/credential routing as model leaf Activities.
container.compaction_service().set_ai_service(container.ai_service())
# Create shared session and activities
session = await create_shared_session(pool_size)
activities = NodeExecutionActivities(session)
from services.temporal.activities import (
broadcast_trigger_status_activity,
evaluate_trigger_filter_activity,
load_persisted_workflow_graph_activity,
pause_workflow_on_failure_activity,
record_run_completion_activity,
store_node_output_activity,
)
from services.temporal.agent_activities import collect_agent_activities
from services.temporal.pi_activities import collect_pi_activities, PiControlActivities
from services.temporal.pi_inline_delegation import collect_pi_inline_activities
from services.temporal.workspace_task_activities import reset_workspace_task_runtime
from services.chat.activities import CHAT_RUN_ACTIVITIES
from services.approvals.activities import APPROVAL_ACTIVITIES
from services.temporal.agent_workflow import (
AgentWorkflow,
DelegatedTaskWorkflow,
)
from services.temporal.plugin_activities import (
collect_plugin_activities,
)
registered_agent_activities = [
*collect_agent_activities(),
*collect_pi_activities(),
*collect_pi_inline_activities(),
*PiControlActivities(client).activities(),
]
registered_plugin_activities = _generic_plugin_activities()
from services.temporal.browser_task_activities import HISTORY_ACTIVITIES, BROWSER_OWNER_ACTIVITIES
from core.config import Settings
local_browser_activities = [] if getattr(Settings(), "distributed_mode", False) is True else BROWSER_OWNER_ACTIVITIES
# No socket follows a chat here: its events go to the backend's hub.
from services.chat.relay import start_relay, stop_relay
start_relay()
pi_driver_pool = PiDriverWorkerPool(client)
try:
await pi_driver_pool.start()
worker = Worker(
client,
task_queue=task_queue,
workflows=_framework_workflows(),
activities=[
activities.execute_node_activity,
*ExecutionControlActivities(client).activities(),
broadcast_trigger_status_activity,
evaluate_trigger_filter_activity,
load_persisted_workflow_graph_activity,
pause_workflow_on_failure_activity,
record_run_completion_activity,
reset_workspace_task_runtime,
*HISTORY_ACTIVITIES,
*local_browser_activities,
*CHAT_RUN_ACTIVITIES,
*APPROVAL_ACTIVITIES,
store_node_output_activity,
*registered_plugin_activities,
*registered_agent_activities,
],
max_concurrent_activities=pool_size,
max_concurrent_workflow_tasks=10,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
# ``TracingInterceptor`` is registered once on the shared client;
# the SDK prepends client interceptors to every worker, so
# repeating it here double-instruments each Signal / Query /
# Update / Activity and doubles the emitted spans.
interceptors=[ObservabilityWorkerInterceptor()],
)
logger.info("Worker running. Press Ctrl+C to stop.")
await worker.run()
finally:
await pi_driver_pool.stop()
await stop_relay()
# Cleanup session on shutdown
if not session.closed:
await session.close()
# Activities resolve the singleton unifier through the global
# container. Close its pooled SDK clients in standalone workers too.
from core.container import container
await container.chat_unifier().aclose()
async def create_worker(
client: Client,
task_queue: str = "machina-tasks",
session: Optional[aiohttp.ClientSession] = None,
) -> Worker:
"""Create a worker instance for use in tests or custom setups.
Args:
client: Connected Temporal client
task_queue: Task queue name
session: Optional shared aiohttp session (created if not provided)
Returns:
Configured Worker instance (not started)
Coding Agent setups must also start ``create_pi_driver_worker(client)``;
the separate worker is required to keep driver waits out of leaf slots.
"""
if session is None:
session = await create_shared_session()
activities = NodeExecutionActivities(session)
from services.temporal.activities import (
broadcast_trigger_status_activity,
evaluate_trigger_filter_activity,
load_persisted_workflow_graph_activity,
pause_workflow_on_failure_activity,
record_run_completion_activity,
store_node_output_activity,
)
from services.temporal.agent_activities import collect_agent_activities
from services.temporal.pi_activities import collect_pi_activities, PiControlActivities
from services.temporal.pi_inline_delegation import collect_pi_inline_activities
from services.temporal.workspace_task_activities import reset_workspace_task_runtime
from services.chat.activities import CHAT_RUN_ACTIVITIES
from services.approvals.activities import APPROVAL_ACTIVITIES
from services.temporal.agent_workflow import (
AgentWorkflow,
DelegatedTaskWorkflow,
)
from services.temporal.plugin_activities import (
collect_plugin_activities,
)
registered_agent_activities = [
*collect_agent_activities(),
*collect_pi_activities(),
*collect_pi_inline_activities(),
*PiControlActivities(client).activities(),
]
registered_plugin_activities = _generic_plugin_activities()
from services.temporal.browser_task_activities import HISTORY_ACTIVITIES, BROWSER_OWNER_ACTIVITIES
from core.config import Settings
local_browser_activities = [] if getattr(Settings(), "distributed_mode", False) is True else BROWSER_OWNER_ACTIVITIES
return Worker(
client,
task_queue=task_queue,
workflows=_framework_workflows(),
activities=[
activities.execute_node_activity,
*ExecutionControlActivities(client).activities(),
broadcast_trigger_status_activity,
evaluate_trigger_filter_activity,
load_persisted_workflow_graph_activity,
pause_workflow_on_failure_activity,
record_run_completion_activity,
reset_workspace_task_runtime,
*HISTORY_ACTIVITIES,
*local_browser_activities,
*CHAT_RUN_ACTIVITIES,
*APPROVAL_ACTIVITIES,
store_node_output_activity,
*registered_plugin_activities,
*registered_agent_activities,
],
max_concurrent_activities=100,
max_concurrent_workflow_tasks=10,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
# ``TracingInterceptor`` is registered once on the shared client;
# the SDK prepends client interceptors to every worker, so
# repeating it here double-instruments each Signal / Query /
# Update / Activity and doubles the emitted spans.
interceptors=[ObservabilityWorkerInterceptor()],
)
if __name__ == "__main__":
# Allow running worker standalone
# Usage: python -m services.temporal.worker
from core.config import Settings
from core.logging import configure_logging
# The embedded worker logs through main.py's configuration; a standalone
# one needs its own (levels, and the filters on the SDK's loggers).
configure_logging(Settings())
asyncio.run(run_standalone_worker())