Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Acknowledged generation control using native Temporal execution trees.
The controller records only roots which can outlive a parent. Descriptions
discover attached children after each workflow has fenced child starts.
No visibility scan is used as evidence that a generation is quiescent.
"""
from __future__ import annotations
import asyncio
import time
from typing import Any
from temporalio.client import WorkflowExecutionStatus
from temporalio.service import RPCError, RPCStatusCode
from core.logging import get_logger
logger = get_logger(__name__)
class GenerationTransitionSuperseded(RuntimeError):
"""A newer durable control intent owns this generation."""
#: Children that take no part in generation control: they admit no actions
#: and answer no control Updates or Queries. An approval wait
#: (services/temporal/approval_workflow.py) waits for the owner, so Stop
#: completes around it.
NON_PARTICIPANTS = frozenset({"ApprovalWorkflow"})
def _participants(desc: Any) -> list:
return [child for child in desc.raw_description.pending_children if child.workflow_type_name not in NON_PARTICIPANTS]
async def count_generation_actions(client: Any, controller_id: str, status: dict) -> int:
"""A read-only UI count; transition verification uses acknowledgements."""
queue = [controller_id, *(status.get("live_roots") or {})]
seen: set[str] = set()
count = 0
while queue:
workflow_id = queue.pop()
if workflow_id in seen:
continue
seen.add(workflow_id)
handle = client.get_workflow_handle(workflow_id)
desc = await handle.describe()
if desc.status != WorkflowExecutionStatus.RUNNING:
continue
participant = await handle.query("execution_control_status")
count += int(participant.get("active_actions", 0))
queue.extend(child.workflow_id for child in _participants(desc))
return count
async def transition_generation(client: Any, control: Any, *, paused: bool, on_admission: Any = None,
ensure_current: Any = None) -> dict:
controller = client.get_workflow_handle(control.controller_workflow_id)
revision = int(control.revision)
base = {"generation": int(control.generation), "revision": revision,
"state": "paused" if paused else "running"}
started = time.monotonic()
async def current() -> None:
if ensure_current is not None:
await ensure_current()
async def describe(workflow_id: str, chain: str) -> Any:
await current()
handle = client.get_workflow_handle(workflow_id)
try:
desc = await handle.describe()
except RPCError as exc:
if exc.status == RPCStatusCode.NOT_FOUND:
return None
raise
actual = desc.raw_description.workflow_execution_info.first_run_id
if chain and actual != chain:
# Workflow ID reuse must never let an old root control a new chain.
return None
if desc.status != WorkflowExecutionStatus.RUNNING:
return None
return desc
controller_description = await describe(control.controller_workflow_id, control.controller_run_id or "")
if controller_description is None:
raise RuntimeError("controller_execution_missing")
controller_chain = controller_description.raw_description.workflow_execution_info.first_run_id
controller_payload = {**base, "first_execution_run_id": controller_chain, "producers_held": True}
await current()
status = await controller.execute_update("set_control_state", controller_payload,
id=f"control:{revision}:admission")
if status.get("participant_revision", status.get("revision")) != revision:
raise RuntimeError("control_revision_conflict")
targets: dict[str, str] = {}
order: list[str] = []
removed: set[tuple[str, str]] = set()
while True:
await current()
before = await controller.query("status")
epoch = before["membership_epoch"]
roots = before.get("live_roots", {})
queue = [(control.controller_workflow_id, controller_chain)]
queue.extend((r["workflow_id"], r["first_execution_run_id"]) for r in roots.values())
visited: set[str] = set()
# First fence every reachable workflow. Never await its activity drain
# here: an unfenced sibling could otherwise continue admitting work.
while queue:
workflow_id, chain = queue.pop()
if workflow_id in visited:
continue
visited.add(workflow_id)
desc = await describe(workflow_id, chain)
if desc is None:
targets.pop(workflow_id, None)
if workflow_id in roots and (workflow_id, chain) not in removed:
await current()
await controller.execute_update("unregister_execution", {
"workflow_id": workflow_id, "first_execution_run_id": chain,
"generation": control.generation,
})
removed.add((workflow_id, chain))
continue
current_chain = desc.raw_description.workflow_execution_info.first_run_id
handle = client.get_workflow_handle(workflow_id)
if workflow_id != control.controller_workflow_id:
payload = {**base, "first_execution_run_id": current_chain, "producers_held": True}
try:
await current()
ack = await handle.execute_update("set_control_state", payload,
id=f"control:{revision}:admission")
except RPCError as exc:
if exc.status == RPCStatusCode.NOT_FOUND:
continue
raise
if ack.get("revision") != revision or ack.get("state") != base["state"]:
raise RuntimeError("temporal_control_ack_mismatch")
targets[workflow_id] = current_chain
if workflow_id not in order:
order.append(workflow_id)
# Setter acknowledges admitted child starts before this second
# describe, so pending_children is a complete direct-child fence.
desc = await describe(workflow_id, current_chain)
if desc is not None:
queue.extend((child.workflow_id, child.run_id) for child in _participants(desc))
logger.info("Generation admission acknowledged", workflow_id=control.workflow_id,
revision=revision, latency_seconds=time.monotonic() - started,
controlled_executions=len(targets))
if on_admission is not None:
await current()
await on_admission(await controller.query("status"))
if paused:
async def checkpoint(workflow_id: str, chain: str) -> None:
if await describe(workflow_id, chain) is None:
return
try:
await current()
result = await client.get_workflow_handle(workflow_id).execute_update(
"wait_for_checkpoint", {**base, "first_execution_run_id": chain},
id=f"control:{revision}:checkpoint")
except RPCError as exc:
if exc.status == RPCStatusCode.NOT_FOUND:
return
raise
if (not result.get("checkpoint") or result.get("state") != base["state"]
or result.get("revision", result.get("participant_revision")) != revision):
raise RuntimeError("temporal_checkpoint_ack_mismatch")
checkpoints = [asyncio.create_task(checkpoint(wid, chain)) for wid, chain in targets.items()]
try:
await asyncio.gather(*checkpoints)
finally:
# These tasks only poll Updates; cancelling a caller does not
# cancel the admitted Workflow actions or their durable intent.
for task in checkpoints:
if not task.done():
task.cancel()
await asyncio.gather(*checkpoints, return_exceptions=True)
await current()
after = await controller.query("status")
if after["membership_epoch"] != epoch:
continue
if not paused:
for workflow_id in reversed(order):
if workflow_id == control.controller_workflow_id:
continue
chain = targets.get(workflow_id)
if chain and await describe(workflow_id, chain) is not None:
await current()
ack = await client.get_workflow_handle(workflow_id).execute_update(
"set_control_state", {**base, "first_execution_run_id": chain, "producers_held": False},
id=f"control:{revision}:release")
if ack.get("revision") != revision or ack.get("state") != "running" or ack.get("producers_held"):
raise RuntimeError("temporal_control_ack_mismatch")
await current()
status = await controller.execute_update("set_control_state", {
**controller_payload, "producers_held": False,
"expected_membership_epoch": epoch,
})
if status.get("participant_revision") != revision or status.get("participant_state") != "running":
raise RuntimeError("control_revision_conflict")
if status.get("producers_held"):
continue
else:
status = after
if status["membership_epoch"] == epoch:
break
logger.info("Generation checkpoint acknowledged", workflow_id=control.workflow_id,
revision=revision, latency_seconds=time.monotonic() - started,
controlled_executions=len(targets))
return {"controller_status": status, "controlled_executions": len(targets)}