Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Future snapshots change without erasing an employee's runtime data."""
from copy import deepcopy
import asyncio
from datetime import datetime, timedelta, timezone
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from sqlmodel import select
from models.agent_context import AgentConversation
from models.database import AgentTeam, TeamTask, WorkflowControlExecution
from models.employees import EmployeeApply, EmployeeJob
from services.deployment.state import DeploymentState
from services.employees import safe_apply
REAL_ACTIVE = safe_apply._active
class EmptyAsync:
def __aiter__(self): return self
async def __anext__(self): raise StopAsyncIteration
@pytest.fixture
async def harness(real_database, monkeypatch):
import core.container as container_module
import services.deployment.handlers as handlers
import services.workflow_validator as validator
before = {"owner_id": "owner", "graphVersion": 2, "nodes": [{"id": "7:aiAgent:1", "type": "aiAgent", "data": {"label": "Worker"}}], "edges": []}
after = {**deepcopy(before), "nodes": [*before["nodes"], {"id": "7:calculatorTool:1", "type": "calculatorTool", "data": {"label": "Calculate"}}]}
await real_database.save_workflow("7", "Maya", "Maya_7", after)
async with real_database.get_session() as session:
session.add(WorkflowControlExecution(id="control", workflow_id="7", generation=1, execution_id="execution", root_execution_id="execution", data_scope_id="scope", status="running", revision=1, idempotency_key="start", graph_hash="old", graph_snapshot=before))
session.add(AgentConversation(workflow_id="7", generation=1, agent_node_id="7:aiAgent:1", messages=[{"role": "user", "content": "Remember this"}]))
await session.commit()
state = SimpleNamespace(updates=[], update_ids=[], active=False, fail=False, paused=False)
class Handle:
async def execute_update(self, name, argument, **kwargs):
state.updates.append((name, deepcopy(argument)))
state.update_ids.append((name, deepcopy(argument), kwargs.get("id")))
if name == "replace_graph" and state.fail:
state.fail = False
raise RuntimeError("handoff failed")
return {"applied": True}
class Manager:
def __init__(self):
self._deployments = {"7": DeploymentState("deployment", "7", True, before["nodes"], [], "scope")}
self._paused_events = {"7": [("trigger", {"id": "accepted"})]}
self._active_runs = {}
def pause(self, _workflow_id): state.paused = True
async def resume(self, _workflow_id): state.paused = False
manager = Manager()
client = SimpleNamespace(list_schedules=AsyncMock(return_value=EmptyAsync()))
service = SimpleNamespace(_get_deployment_manager=lambda: manager)
container = SimpleNamespace(settings=lambda: SimpleNamespace(temporal_enabled=True), temporal_client=lambda: SimpleNamespace(client=client), workflow_service=lambda: service)
monkeypatch.setattr(container_module, "container", container)
monkeypatch.setattr(handlers, "_controller_handle", lambda _control: Handle())
state.real_cron_pause = handlers._set_cron_pause
monkeypatch.setattr(handlers, "_set_cron_pause", AsyncMock(return_value=0))
monkeypatch.setattr(validator, "validate_workflow", AsyncMock(return_value={"errors": []}))
monkeypatch.setattr(safe_apply, "_active", AsyncMock(side_effect=lambda *_args: state.active))
import services.employees.events as events
monkeypatch.setattr(events, "employee_changed_now", lambda *_args: None)
state.database, state.before, state.after, state.manager, state.client = real_database, before, after, manager, client
state.operational = {**deepcopy(after), "parameters": {node["id"]: {} for node in after["nodes"]}}
state.container, state.Handle = container, Handle
return state
async def test_apply_preserves_context_namespace_and_accepted_events(harness):
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="one")
assert result == {"success": True, "activation_state": "running"}
control = await harness.database.get_latest_workflow_control("7")
assert control.graph_snapshot == harness.operational and control.generation == 1 and control.data_scope_id == "scope"
assert harness.manager._paused_events["7"] == [("trigger", {"id": "accepted"})]
async with harness.database.get_session() as session:
conversation = (await session.execute(select(AgentConversation))).scalar_one()
assert conversation.messages[0]["content"] == "Remember this"
assert [name for name, _ in harness.updates] == ["pause_admissions", "replace_graph", "set_control_state"]
async def test_active_work_defers_handoff_and_recovers(harness):
harness.active = True
waiting = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="one")
assert waiting["activation_state"] == "waiting"
assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == harness.before
harness.active = False
await safe_apply.recover_applies(harness.database)
assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == harness.operational
async def test_paused_employee_stays_paused(harness):
async with harness.database.get_session() as session:
control = await session.get(WorkflowControlExecution, "control")
control.status = "paused"
await session.commit()
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="one")
assert result["activation_state"] == "paused" and harness.paused
assert not any(name == "set_control_state" for name, _ in harness.updates)
async def test_failed_handoff_keeps_operational_snapshot(harness):
harness.fail = True
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="one")
assert result["error"] == "apply_failed"
control = await harness.database.get_latest_workflow_control("7")
assert control.graph_snapshot == harness.before and control.data_scope_id == "scope"
assert (await harness.database.get_workflow("7")).data == harness.after
assert harness.manager._deployments["7"].nodes == harness.before["nodes"]
async def test_foreign_owner_cannot_apply(harness):
assert (await safe_apply.apply_saved_changes(harness.database, "7", owner_id="other", key="one"))["error"] == "not_found"
assert harness.updates == []
class ScheduleHandle:
def __init__(self, schedule):
self.schedule = schedule
self.updates = []
async def describe(self):
return SimpleNamespace(schedule=self.schedule)
async def update(self, updater):
from inspect import isawaitable
from temporalio.client import ScheduleUpdateInput
result = updater(ScheduleUpdateInput(description=SimpleNamespace(schedule=self.schedule)))
if isawaitable(result):
result = await result
self.updates.append(result.schedule)
self.schedule = result.schedule
async def pause(self, *, note):
self.schedule = replace(self.schedule, state=replace(self.schedule.state, paused=True, note=note))
async def unpause(self, *, note):
self.schedule = replace(self.schedule, state=replace(self.schedule.state, paused=False, note=note))
async def encoded_schedule(harness):
"""Use SDK Payloads as describe() returns, not already decoded test dictionaries."""
from temporalio.client import Schedule, ScheduleActionStartWorkflow, ScheduleSpec, ScheduleState, SchedulePolicy
from temporalio.converter import DataConverter
converter = DataConverter.default
arguments = await converter.encode([{"nodes": harness.before["nodes"], "edges": [], "owner_id": "owner", "recipient": "original"}, {"retained": "second argument"}, 17])
schedule = Schedule(action=ScheduleActionStartWorkflow("EventWorkflow", args=arguments, id="scheduled-job", task_queue="worker", memo={"owner": "owner"}),
spec=ScheduleSpec(cron_expressions=["15 9 * * 1-5"], time_zone_name="Asia/Kolkata", jitter=timedelta(seconds=7)),
state=ScheduleState(paused=True, note="Owner paused this schedule", limited_actions=True, remaining_actions=3),
policy=SchedulePolicy(catchup_window=timedelta(hours=2), pause_on_failure=True))
handle = ScheduleHandle(schedule)
class ScheduleList:
def __aiter__(self):
async def items():
yield SimpleNamespace(id="owner-schedule")
return items()
harness.client.data_converter = converter
harness.client.list_schedules = AsyncMock(return_value=ScheduleList())
harness.client.get_schedule_handle = lambda _id: handle
return schedule, handle
async def test_sdk_encoded_schedule_keeps_spec_pause_policy_and_extra_arguments(harness):
before, schedule = await encoded_schedule(harness)
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="encoded")
assert result["success"] is True
after = schedule.schedule
assert after.spec == before.spec and after.state == before.state and after.policy == before.policy
assert after.action.args[0] == {"nodes": harness.after["nodes"], "edges": harness.after["edges"], "owner_id": "owner", "recipient": "original", "parameter_snapshot": harness.operational["parameters"]}
assert after.action.args[1:] == [{"retained": "second argument"}, 17]
assert after.action.id == before.action.id and after.action.task_queue == before.action.task_queue
assert after.action.memo == before.action.memo
async def test_failed_handoff_rolls_back_real_encoded_schedule(harness):
before, schedule = await encoded_schedule(harness)
harness.fail = True
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="rollback")
assert result["error"] == "apply_failed"
assert len(schedule.updates) >= 2 and schedule.schedule == before
assert schedule.updates[0].action.args[0]["nodes"] == harness.after["nodes"]
decoded = await harness.client.data_converter.decode(schedule.schedule.action.args)
assert decoded[0]["nodes"] == harness.before["nodes"]
assert decoded[1:] == [{"retained": "second argument"}, 17]
@pytest.mark.parametrize("failure", [False, True])
async def test_handoff_with_real_pause_sweep_preserves_owners_schedule_pause(harness, monkeypatch, failure):
import services.deployment.handlers as handlers
before, schedule = await encoded_schedule(harness)
harness.fail = failure
monkeypatch.setattr(handlers, "_set_cron_pause", harness.real_cron_pause)
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="owner-pause")
assert result["success"] is not failure
assert schedule.schedule.state == before.state
assert schedule.schedule.spec == before.spec and schedule.schedule.policy == before.policy
async def test_restart_during_drain_keeps_original_schedule_owner_edits(harness, monkeypatch):
import services.deployment.handlers as handlers
before, schedule = await encoded_schedule(harness)
monkeypatch.setattr(handlers, "_set_cron_pause", harness.real_cron_pause)
harness.active = True
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="drain-pause")
assert result["activation_state"] == "waiting"
harness.active = False
await safe_apply.recover_applies(harness.database)
assert schedule.schedule.state == before.state
assert schedule.schedule.spec == before.spec and schedule.schedule.policy == before.policy
async def test_stop_work_cancels_durable_records_but_waits_for_actual_execution_root(harness, monkeypatch):
import services.chat.ledger as ledger
monkeypatch.setattr(ledger, "live_runs", AsyncMock(return_value=[]))
monkeypatch.setattr(safe_apply, "_active", REAL_ACTIVE)
harness.container.workflow_service().get_deployment_status = lambda _id: {"active_runs": 0}
roots = {"running": True}
cancelled = []
class RunningRoots:
def __aiter__(self):
async def items():
yield SimpleNamespace(id="controller", workflow_type="WorkflowControlWorkflow")
if roots["running"]:
yield SimpleNamespace(id="job-root", workflow_type="EventWorkflow")
return items()
async def cancel():
cancelled.append("job-root")
# Temporal cancellation is a request; the execution has not closed yet.
harness.client.list_workflows = lambda **_kwargs: RunningRoots()
harness.client.get_workflow_handle = lambda _id: SimpleNamespace(cancel=cancel)
async with harness.database.get_session() as session:
session.add(AgentTeam(id="team", workflow_id="7", team_lead_node_id="7:aiAgent:1"))
for status in ("blocked", "queued", "pending", "running", "accepted"):
session.add(TeamTask(id=f"task-{status}", team_id="team", title="Research", created_by="7:aiAgent:1", workflow_id="7", status=status))
session.add(EmployeeJob(id="job", workflow_id="7", origin_execution_id="origin", lead_node_id="7:aiAgent:1", team_id="team", state="working"))
await session.commit()
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="stop", stop_work=True)
assert result["activation_state"] == "waiting" and cancelled == ["job-root"]
async with harness.database.get_session() as session:
assert (await session.get(EmployeeJob, "job")).state == "cancelled"
assert (await session.get(AgentTeam, "team")).status == "dissolved"
for status in ("blocked", "queued", "pending", "running"):
task = await session.get(TeamTask, f"task-{status}")
assert task.status == "cancelled" and task.cancellation_requested
assert (await session.get(TeamTask, "task-accepted")).status == "accepted"
assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == harness.before
roots["running"] = False
await safe_apply.recover_applies(harness.database)
assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == harness.operational
assert harness.manager._paused_events["7"] == [("trigger", {"id": "accepted"})]
async def test_live_claim_serializes_handoffs_and_renews_lease(harness, monkeypatch):
import services.deployment.handlers as handlers
entered, release, tick, renewed = (asyncio.Event() for _ in range(4))
original_sleep = asyncio.sleep
calls = 0
async def sleep(seconds):
nonlocal calls
if seconds != 30:
return await original_sleep(seconds)
calls += 1
if calls == 1:
await tick.wait()
else:
renewed.set()
await asyncio.Event().wait()
monkeypatch.setattr(safe_apply.asyncio, "sleep", sleep)
class SlowHandle(harness.Handle):
async def execute_update(self, name, argument, **kwargs):
result = await super().execute_update(name, argument, **kwargs)
if name == "replace_graph":
entered.set()
await release.wait()
return result
monkeypatch.setattr(handlers, "_controller_handle", lambda _control: SlowHandle())
first = asyncio.create_task(safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="lease"))
try:
await asyncio.wait_for(entered.wait(), timeout=10)
async with harness.database.reserved_session() as session:
claim = await session.get(EmployeeApply, "apply:7:lease")
token = claim.claim_token
claim.lease_until = datetime.now(timezone.utc) + timedelta(seconds=10)
await session.commit()
tick.set()
await asyncio.wait_for(renewed.wait(), timeout=10)
async with harness.database.get_session() as session:
claim = await session.get(EmployeeApply, "apply:7:lease")
assert claim.claim_token == token
assert claim.lease_until.replace(tzinfo=timezone.utc) > datetime.now(timezone.utc) + timedelta(minutes=1)
retry = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="lease")
assert retry["activation_state"] == "starting"
assert (await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="other"))["error"] == "conflict"
assert sum(name == "replace_graph" for name, _ in harness.updates) == 1
finally:
release.set()
await first
async def test_close_open_work_leaves_nothing_for_apply_to_wait_on(harness, monkeypatch):
# 1b.21: Reset closes the generation's open work as Stop work does, and its
# queued requests too.
import services.chat.ledger as ledger
monkeypatch.setattr(ledger, "live_runs", AsyncMock(return_value=[]))
harness.container.workflow_service().get_deployment_status = lambda _id: {"active_runs": 0}
harness.client.list_workflows = lambda **_kwargs: EmptyAsync()
closed_tasks, kept_tasks = ("pending", "blocked", "queued", "running"), ("submitted", "accepted", "failed", "cancelled")
open_jobs, kept_jobs = ("running", "working", "delivering", "delivery_queued", "queued"), ("delivered", "failed", "cancelled", "delivery_needs_review")
async with harness.database.get_session() as session:
session.add(AgentTeam(id="team", workflow_id="7", team_lead_node_id="7:aiAgent:1"))
session.add(AgentTeam(id="done-team", workflow_id="7", team_lead_node_id="7:aiAgent:1", status="completed"))
for status in (*closed_tasks, *kept_tasks):
session.add(TeamTask(id=f"task-{status}", team_id="team", title="Research", created_by="7:aiAgent:1", workflow_id="7", status=status))
for state in (*open_jobs, *kept_jobs):
delivery = {"inflight": "send"} if state == "delivering" else {}
session.add(EmployeeJob(id=f"job-{state}", workflow_id="7", origin_execution_id=f"origin-{state}", lead_node_id="7:aiAgent:1",
team_id="team", state=state, delivery=delivery))
await session.commit()
assert await REAL_ACTIVE(harness.database, "7")
for _ in range(2): # idempotent: Reset may run again
await safe_apply.close_open_work(harness.database, "7", reason="The workflow was reset",
job_states=(*safe_apply.OPEN_JOB_STATES, "queued"))
assert not await REAL_ACTIVE(harness.database, "7")
async with harness.database.get_session() as session:
assert (await session.get(AgentTeam, "team")).status == "dissolved"
assert (await session.get(AgentTeam, "done-team")).status == "completed"
for status in closed_tasks:
assert (await session.get(TeamTask, f"task-{status}")).status == "cancelled"
for status in kept_tasks:
assert (await session.get(TeamTask, f"task-{status}")).status == status
for state in open_jobs:
assert (await session.get(EmployeeJob, f"job-{state}")).state == "cancelled"
for state in kept_jobs:
assert (await session.get(EmployeeJob, f"job-{state}")).state == state
# The in-flight send mark is kept for the owner to resolve.
assert (await session.get(EmployeeJob, "job-delivering")).delivery["inflight"] == "send"
async def test_stop_work_keeps_queued_requests(harness):
async with harness.database.get_session() as session:
session.add(EmployeeJob(id="job-queued", workflow_id="7", origin_execution_id="origin", lead_node_id="7:aiAgent:1", state="queued"))
await session.commit()
await safe_apply.close_open_work(harness.database, "7", reason="The owner stopped work to apply new abilities")
async with harness.database.get_session() as session:
assert (await session.get(EmployeeJob, "job-queued")).state == "queued"
def _hang_on(harness, monkeypatch, hangs):
"""1b.15: the controller never answers the Updates ``hangs`` picks."""
import services.deployment.handlers as handlers
class HangingHandle(harness.Handle):
async def execute_update(self, name, argument, **kwargs):
result = await super().execute_update(name, argument, **kwargs)
if hangs(name, kwargs.get("id") or ""):
await asyncio.Event().wait()
return result
monkeypatch.setattr(safe_apply, "UPDATE_SECONDS", 0.05)
monkeypatch.setattr(handlers, "_controller_handle", lambda _control: HangingHandle())
def _answer_again(harness, monkeypatch):
import services.deployment.handlers as handlers
monkeypatch.setattr(handlers, "_controller_handle", lambda _control: harness.Handle())
async def _request(harness, key):
async with harness.database.get_session() as session:
return await session.get(EmployeeApply, f"apply:7:{key}")
async def test_an_unanswered_graph_update_waits_without_rollback_and_recovery_applies_it(harness, monkeypatch):
_hang_on(harness, monkeypatch, lambda name, _id: name == "replace_graph")
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="slow")
assert result["success"] is True and result["activation_state"] == "waiting"
assert not any(":rollback" in (update_id or "") for _name, _argument, update_id in harness.update_ids)
assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == harness.before
assert harness.manager._deployments["7"].nodes == harness.before["nodes"]
request = await _request(harness, "slow")
assert request.state == "applying" and request.claim_token
# The claim lapses with its lease; recovery applies the snapshot again.
async with harness.database.reserved_session() as session:
(await session.get(EmployeeApply, "apply:7:slow")).lease_until = datetime.now(timezone.utc) - timedelta(minutes=1)
await session.commit()
_answer_again(harness, monkeypatch)
await safe_apply.recover_applies(harness.database)
assert (await _request(harness, "slow")).state == "applied"
assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == harness.operational
async def test_an_unanswered_admission_pause_keeps_the_apply_waiting_and_recovery_bounded(harness, monkeypatch):
_hang_on(harness, monkeypatch, lambda name, _id: name == "pause_admissions")
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="first")
assert result["activation_state"] == "waiting"
assert (await _request(harness, "first")).state == "waiting"
# The recovery loop's own call answers too, instead of stalling the loop.
await asyncio.wait_for(safe_apply.recover_applies(harness.database), timeout=5)
assert (await _request(harness, "first")).state == "waiting"
_answer_again(harness, monkeypatch)
await safe_apply.recover_applies(harness.database)
assert (await _request(harness, "first")).state == "applied"
async def test_an_unanswered_rollback_leaves_the_apply_waiting_not_failed(harness, monkeypatch):
harness.fail = True # the forward replace_graph fails once
_hang_on(harness, monkeypatch, lambda _name, update_id: ":rollback-pause:" in update_id)
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="undo")
assert result["activation_state"] == "waiting"
assert (await _request(harness, "undo")).state == "applying"
async def test_restart_recovers_expired_applying_lease(harness):
async with harness.database.reserved_session() as session:
session.add(EmployeeApply(id="apply:7:expired", workflow_id="7", owner_id="owner", state="applying", resume_after=True,
snapshot=harness.after, previous=harness.before, claim_token="old-process", lease_until=datetime.now(timezone.utc) - timedelta(minutes=1)))
await session.commit()
async with harness.database.get_session() as session:
assert (await session.get(EmployeeApply, "apply:7:expired")).producer_states == {}
await safe_apply.recover_applies(harness.database)
async with harness.database.get_session() as session:
request = await session.get(EmployeeApply, "apply:7:expired")
assert request.state == "applied" and request.claim_token is None and request.lease_until is None
assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == harness.after
async def test_stale_worker_cannot_commit_or_roll_back_a_new_claim(harness, monkeypatch):
import services.deployment.handlers as handlers
entered, release = asyncio.Event(), asyncio.Event()
class SlowHandle(harness.Handle):
async def execute_update(self, name, argument, **kwargs):
result = await super().execute_update(name, argument, **kwargs)
if name == "replace_graph":
entered.set()
await release.wait()
return result
monkeypatch.setattr(handlers, "_controller_handle", lambda _control: SlowHandle())
first = asyncio.create_task(safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="fenced"))
newer = {**deepcopy(harness.after), "nodes": [*harness.after["nodes"], {"id": "7:clock:1", "type": "clock", "data": {"label": "Newer ability"}}]}
try:
await asyncio.wait_for(entered.wait(), timeout=10)
async with harness.database.reserved_session() as session:
request = await session.get(EmployeeApply, "apply:7:fenced")
request.claim_token = "new-process"
request.lease_until = datetime.now(timezone.utc) + timedelta(minutes=2)
request.snapshot = newer
control = await session.get(WorkflowControlExecution, "control")
control.graph_snapshot = newer
control.revision += 1
await session.commit()
harness.manager._deployments["7"] = replace(harness.manager._deployments["7"], nodes=newer["nodes"])
release.set()
result = await first
assert result["error"] == "conflict"
assert not any(name == "replace_graph" and argument == harness.before for name, argument in harness.updates)
async with harness.database.get_session() as session:
request = await session.get(EmployeeApply, "apply:7:fenced")
assert request.claim_token == "new-process" and request.state == "applying"
assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == newer
assert harness.manager._deployments["7"].nodes == newer["nodes"]
finally:
release.set()
await first
async def test_failed_same_key_retry_uses_fresh_controller_updates(harness):
harness.fail = True
failed = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="retry")
assert failed["error"] == "apply_failed"
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="retry")
assert result["success"] is True and result["activation_state"] == "running"
attempts = [identity for name, graph, identity in harness.update_ids if name == "replace_graph" and graph == harness.operational]
assert len(attempts) == 2 and attempts[0] != attempts[1]
pauses = [identity for name, _, identity in harness.update_ids if name == "pause_admissions"]
assert len(set(pauses)) == len(pauses)
async with harness.database.get_session() as session:
request = await session.get(EmployeeApply, "apply:7:retry")
assert request.state == "applied" and request.claim_token is None
async def test_manual_pause_during_drain_takes_precedence_over_apply_resume(harness):
harness.active = True
assert (await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="pause"))["activation_state"] == "waiting"
async with harness.database.reserved_session() as session:
current = await session.get(WorkflowControlExecution, "control")
current.status = "paused"
current.revision += 1
await session.commit()
harness.active = False
await safe_apply.recover_applies(harness.database)
async with harness.database.get_session() as session:
request = await session.get(EmployeeApply, "apply:7:pause")
assert request.state == "applied" and request.resume_after is False
control = await harness.database.get_latest_workflow_control("7")
assert control.status == "paused" and control.graph_snapshot == harness.operational
assert not any(name == "set_control_state" for name, _ in harness.updates)
assert harness.paused is True
async def test_claim_is_retained_until_external_resume_finishes(harness, monkeypatch):
import services.deployment.handlers as handlers
observed = []
class ResumeHandle(harness.Handle):
async def execute_update(self, name, argument, **kwargs):
if name == "set_control_state":
async with harness.database.get_session() as session:
request = await session.get(EmployeeApply, "apply:7:resume")
observed.append((request.state, bool(request.claim_token), request.lease_until is not None))
return await super().execute_update(name, argument, **kwargs)
monkeypatch.setattr(handlers, "_controller_handle", lambda _control: ResumeHandle())
assert (await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="resume"))["success"] is True
assert observed == [("applying", True, True)]
async with harness.database.get_session() as session:
request = await session.get(EmployeeApply, "apply:7:resume")
assert request.state == "applied" and request.claim_token is None
async def test_rollback_cannot_clear_a_claim_replaced_during_external_resume(harness, monkeypatch):
import services.deployment.handlers as handlers
newer = {**deepcopy(harness.operational), "nodes": [*harness.after["nodes"], {"id": "7:clock:1", "type": "clock", "data": {"label": "Newer ability"}}]}
class RollbackHandle(harness.Handle):
async def execute_update(self, name, argument, **kwargs):
result = await super().execute_update(name, argument, **kwargs)
if name == "set_control_state":
# The old process resumes after another recovery worker owns the lease.
async with harness.database.reserved_session() as session:
request = await session.get(EmployeeApply, "apply:7:rollback-fenced")
request.claim_token = "new-process"
request.lease_until = datetime.now(timezone.utc) + timedelta(minutes=2)
current = await session.get(WorkflowControlExecution, "control")
current.graph_snapshot = newer
await session.commit()
harness.manager._deployments["7"] = replace(harness.manager._deployments["7"], nodes=newer["nodes"])
return result
harness.fail = True
monkeypatch.setattr(handlers, "_controller_handle", lambda _control: RollbackHandle())
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="rollback-fenced")
assert result["error"] == "conflict"
async with harness.database.get_session() as session:
request = await session.get(EmployeeApply, "apply:7:rollback-fenced")
assert request.state == "applying" and request.claim_token == "new-process"
assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == newer
assert harness.manager._deployments["7"].nodes == newer["nodes"]
async def test_apply_admits_the_employee_team_plan_on_its_lead(harness):
"""The saved graph keeps node data to UI fields; Apply copies the plan in (#148)."""
from models.employees import Employee
plan = {"version": 2, "lead_node_id": "7:ai_employee:1", "talk_node_id": "7:aiAgent:1"}
lead = {"id": "7:ai_employee:1", "type": "ai_employee", "data": {"label": "Lead"}}
saved = {**deepcopy(harness.after), "nodes": [*harness.after["nodes"], lead]}
await harness.database.save_workflow("7", "Maya", "Maya_7", saved)
async with harness.database.get_session() as session:
session.add(Employee(id="employee", workflow_id="7", owner_id="owner", hire_state="ready", team_plan=plan))
await session.commit()
result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="team")
assert result == {"success": True, "activation_state": "running"}
snapshot = (await harness.database.get_latest_workflow_control("7")).graph_snapshot
admitted = next(node for node in snapshot["nodes"] if node["id"] == lead["id"])
assert admitted["data"] == {"label": "Lead", "employee_recipe_version": 2, "employee_team_plan": plan}
assert (await harness.database.get_workflow("7")).data == saved