Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(runtime): make shared node state mutations atomic
trohitg16b15c42e438
diff --git a/server/core/database.py b/server/core/database.pyindex 9082f4da..92685ecd 100644--- a/server/core/database.py+++ b/server/core/database.py@@ -1,18 +1,21 @@"""Modern async database service with SQLModel and SQLAlchemy 2.0."""import json+import inspect+from copy import deepcopyfrom datetime import datetime, timezone-from typing import Dict, Any, List, Optional+from typing import Awaitable, Callable, Dict, Any, List, Optional, Tuple, Unionfrom pydantic_core import to_jsonable_pythonfrom sqlmodel import SQLModel, selectfrom sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmakerfrom sqlalchemy.exc import IntegrityError-from sqlalchemy import text, func+from sqlalchemy import case, text, func, updatefrom contextlib import asynccontextmanagerfrom core.config import Settingsfrom models.database import (NodeParameter,+ RuntimeMutation,Workflow,Execution,APIKey,@@ -40,6 +43,18 @@ from core.logging import get_loggerlogger = get_logger(__name__)+RuntimeMutationCallback = Callable[+ [AsyncSession],+ Union[Optional[Dict[str, Any]], Awaitable[Optional[Dict[str, Any]]]],+]+ParameterMutator = Callable[+ [Dict[str, Any]],+ Union[+ Dict[str, Any],+ Tuple[Dict[str, Any], Optional[Dict[str, Any]]],+ ],+]+class Database:"""Async database service with SQLModel."""@@ -231,6 +246,165 @@ class Database:finally:await session.close()+ async def run_runtime_mutation(+ self,+ *,+ resource_type: str,+ resource_id: str,+ operation: str,+ mutate: RuntimeMutationCallback,+ mutation_id: Optional[str] = None,+ ) -> Tuple[Dict[str, Any], bool]:+ """Run one shared-state mutation under a reserved write transaction.++ SQLite's normal deferred transactions allow two workers to read the+ same JSON value before either writes it. ``BEGIN IMMEDIATE`` obtains+ the write reservation *before* the read, serialising the complete+ read/modify/write cycle. When ``mutation_id`` is supplied, the+ durable ledger row is committed in that same transaction and a retry+ returns the original result with ``applied=False``.++ The callback receives the transaction's :class:`AsyncSession` and+ must not commit it. This method also works with non-SQLite test or+ future production databases, using their regular transaction begin.+ """+ if not self.async_session:+ raise RuntimeError("Database not initialized")++ async with self.async_session() as session:+ try:+ if self.engine is not None and self.engine.dialect.name == "sqlite":+ await session.execute(text("BEGIN IMMEDIATE"))+ else:+ await session.begin()++ if mutation_id:+ found = await session.execute(+ select(RuntimeMutation).where(+ RuntimeMutation.mutation_id == mutation_id,+ RuntimeMutation.resource_type == resource_type,+ RuntimeMutation.resource_id == resource_id,+ )+ )+ existing = found.scalar_one_or_none()+ if existing is not None:+ result = deepcopy(existing.result or {})+ await session.rollback()+ return result, False++ result_or_awaitable = mutate(session)+ if inspect.isawaitable(result_or_awaitable):+ result = await result_or_awaitable+ else:+ result = result_or_awaitable+ result = deepcopy(result or {})++ if mutation_id:+ session.add(+ RuntimeMutation(+ mutation_id=mutation_id,+ resource_type=resource_type,+ resource_id=resource_id,+ operation=operation,+ result=result,+ )+ )++ await session.commit()+ return result, True+ except Exception:+ await session.rollback()+ raise++ async def mutate_node_parameters_atomic(+ self,+ node_id: str,+ mutator: ParameterMutator,+ *,+ mutation_id: Optional[str] = None,+ operation: str = "update",+ ) -> Tuple[Dict[str, Any], Dict[str, Any], bool]:+ """Atomically transform one node's parameters.++ ``mutator`` receives an isolated copy and may return either the new+ parameter dict or ``(new_parameters, result_metadata)``. The final+ persisted parameters, metadata, and whether this call applied the+ write are returned.+ """++ async def _mutate(session: AsyncSession) -> Dict[str, Any]:+ selected = await session.execute(+ select(NodeParameter).where(NodeParameter.node_id == node_id)+ )+ row = selected.scalar_one_or_none()+ current = deepcopy(row.parameters if row is not None else {})+ transformed = mutator(current)+ if isinstance(transformed, tuple):+ new_parameters, metadata = transformed+ else:+ new_parameters, metadata = transformed, {}+ if not isinstance(new_parameters, dict):+ raise TypeError("node parameter mutator must return a dict")+ if row is None:+ session.add(+ NodeParameter(+ node_id=node_id,+ parameters=deepcopy(new_parameters),+ )+ )+ else:+ row.parameters = deepcopy(new_parameters)+ row.updated_at = datetime.now(timezone.utc)+ return deepcopy(metadata or {})++ metadata, applied = await self.run_runtime_mutation(+ resource_type="node_parameters",+ resource_id=node_id,+ operation=operation,+ mutation_id=mutation_id,+ mutate=_mutate,+ )+ parameters = await self.get_node_parameters(node_id) or {}+ return parameters, metadata, applied++ async def mutate_workflow_data_atomic(+ self,+ workflow_id: str,+ mutator: ParameterMutator,+ *,+ mutation_id: Optional[str] = None,+ operation: str = "update",+ ) -> Tuple[Optional[Workflow], Dict[str, Any], bool]:+ """Atomically transform ``Workflow.data`` without stale overwrites."""++ async def _mutate(session: AsyncSession) -> Dict[str, Any]:+ selected = await session.execute(+ select(Workflow).where(Workflow.id == workflow_id)+ )+ workflow = selected.scalar_one_or_none()+ if workflow is None:+ return {"found": False}+ transformed = mutator(deepcopy(workflow.data or {}))+ if isinstance(transformed, tuple):+ new_data, metadata = transformed+ else:+ new_data, metadata = transformed, {}+ if not isinstance(new_data, dict):+ raise TypeError("workflow data mutator must return a dict")+ workflow.data = deepcopy(new_data)+ workflow.updated_at = datetime.now(timezone.utc)+ return {"found": True, **deepcopy(metadata or {})}++ metadata, applied = await self.run_runtime_mutation(+ resource_type="workflow",+ resource_id=workflow_id,+ operation=operation,+ mutation_id=mutation_id,+ mutate=_mutate,+ )+ workflow = await self.get_workflow(workflow_id)+ return workflow, metadata, applied+# ============================================================================# Node Parameters# ============================================================================@@ -2300,60 +2474,146 @@ class Database:return []async def claim_task(self, task_id: str, agent_node_id: str) -> Optional[Dict[str, Any]]:- """Claim a pending task. Returns None if already claimed."""+ """Atomically claim a pending task; same-agent retries are safe."""try:async with self.get_session() as session:- result = await session.execute(- select(TeamTask).where(TeamTask.id == task_id, TeamTask.status == "pending", TeamTask.assigned_to.is_(None))+ claimed = await session.execute(+ update(TeamTask)+ .where(+ TeamTask.id == task_id,+ TeamTask.status == "pending",+ TeamTask.assigned_to.is_(None),+ )+ .values(+ assigned_to=agent_node_id,+ status="in_progress",+ started_at=datetime.now(timezone.utc),+ ))- task = result.scalar_one_or_none()- if not task:+ # The guarded UPDATE is the compare-and-swap. It remains+ # correct on databases where a SELECT + Python mutation would+ # race because there is no SQLite BEGIN IMMEDIATE reservation.+ selected = await session.execute(+ select(TeamTask).where(TeamTask.id == task_id)+ )+ task = selected.scalar_one_or_none()+ if task is None:+ await session.rollback()return None- task.assigned_to = agent_node_id- task.status = "in_progress"- task.started_at = datetime.now(timezone.utc)++ won = bool(claimed.rowcount == 1)+ same_agent_retry = (+ task.status == "in_progress"+ and task.assigned_to == agent_node_id+ )+ if not won and not same_agent_retry:+ await session.rollback()+ return None+await session.commit()- return {"id": task.id, "title": task.title, "assigned_to": agent_node_id}+ return {+ "id": task.id,+ "title": task.title,+ "assigned_to": agent_node_id,+ }except Exception as e:logger.error(f"Failed to claim task: {e}")return Noneasync def complete_task(self, task_id: str, result_data: Optional[Dict[str, Any]] = None) -> bool:- """Mark task as completed."""- try:- async with self.get_session() as session:- result = await session.execute(select(TeamTask).where(TeamTask.id == task_id))- task = result.scalar_one_or_none()- if not task:- return False- task.status = "completed"- task.result = result_data- task.progress = 100- task.completed_at = datetime.now(timezone.utc)- await session.commit()- return True+ """Atomically transition an in-progress task to completed."""+ try:+ async def _complete(session: AsyncSession) -> Dict[str, Any]:+ completed = await session.execute(+ update(TeamTask)+ .where(+ TeamTask.id == task_id,+ TeamTask.status == "in_progress",+ )+ .values(+ status="completed",+ result=result_data,+ progress=100,+ completed_at=datetime.now(timezone.utc),+ )+ )+ if completed.rowcount == 1:+ return {"success": True}++ selected = await session.execute(+ select(TeamTask).where(TeamTask.id == task_id)+ )+ task = selected.scalar_one_or_none()+ return {+ "success": bool(+ task is not None and task.status == "completed"+ )+ }++ result, _applied = await self.run_runtime_mutation(+ resource_type="team_task",+ resource_id=task_id,+ operation="complete",+ mutate=_complete,+ )+ return bool(result.get("success"))except Exception as e:logger.error(f"Failed to complete task: {e}")return Falseasync def fail_task(self, task_id: str, error: str) -> bool:- """Mark task as failed."""- try:- async with self.get_session() as session:- result = await session.execute(select(TeamTask).where(TeamTask.id == task_id))- task = result.scalar_one_or_none()- if not task:- return False- task.error = error- task.retry_count += 1- if task.retry_count < task.max_retries:- task.status = "pending"- task.assigned_to = None- else:- task.status = "failed"- task.completed_at = datetime.now(timezone.utc)- await session.commit()- return True+ """Atomically record one failed attempt and release or fail it."""+ try:+ async def _fail(session: AsyncSession) -> Dict[str, Any]:+ next_retry = TeamTask.retry_count + 1+ retryable = next_retry < TeamTask.max_retries+ failed = await session.execute(+ update(TeamTask)+ .where(+ TeamTask.id == task_id,+ TeamTask.status == "in_progress",+ )+ .values(+ error=error,+ retry_count=next_retry,+ status=case(+ (retryable, "pending"),+ else_="failed",+ ),+ assigned_to=case(+ (retryable, None),+ else_=TeamTask.assigned_to,+ ),+ completed_at=case(+ (retryable, TeamTask.completed_at),+ else_=datetime.now(timezone.utc),+ ),+ )+ )+ if failed.rowcount == 1:+ return {"success": True}++ selected = await session.execute(+ select(TeamTask).where(TeamTask.id == task_id)+ )+ task = selected.scalar_one_or_none()+ # A transport retry after this exact failure observes the+ # released/terminal state and must not increment twice.+ return {+ "success": bool(+ task is not None+ and task.status in {"pending", "failed"}+ and task.error == error+ )+ }++ result, _applied = await self.run_runtime_mutation(+ resource_type="team_task",+ resource_id=task_id,+ operation="fail",+ mutate=_fail,+ )+ return bool(result.get("success"))except Exception as e:logger.error(f"Failed to fail task: {e}")return Falsediff --git a/server/models/database.py b/server/models/database.pyindex 25801053..ee0f9736 100644--- a/server/models/database.py+++ b/server/models/database.py@@ -22,6 +22,31 @@ class NodeParameter(SQLModel, table=True):)+class RuntimeMutation(SQLModel, table=True):+ """Durable idempotency ledger for shared runtime state mutations.++ Runtime workers (including Temporal activity retries) can execute the+ same logical write more than once. Keeping the mutation key in the+ same SQLite transaction as the state change makes those retries a+ no-op, even across processes or server restarts.+ """++ __tablename__ = "runtime_mutations"++ # The idempotency identity is scoped to the mutated resource. A caller+ # may legitimately reuse a transport-level mutation id for two different+ # nodes; those writes must not suppress one another.+ mutation_id: str = Field(primary_key=True, max_length=512)+ resource_type: str = Field(primary_key=True, max_length=100)+ resource_id: str = Field(primary_key=True, max_length=512)+ operation: str = Field(default="update", max_length=100)+ result: Optional[Dict[str, Any]] = Field(default=None, sa_column=Column(JSON))+ created_at: datetime = Field(+ default_factory=lambda: datetime.now(timezone.utc),+ sa_column=Column(DateTime(timezone=True), server_default=func.now()),+ )++class Workflow(SQLModel, table=True):"""Workflow definitions.diff --git a/server/nodes/agent/claude_code_agent/__init__.py b/server/nodes/agent/claude_code_agent/__init__.pyindex c55309f8..d17eec14 100644--- a/server/nodes/agent/claude_code_agent/__init__.py+++ b/server/nodes/agent/claude_code_agent/__init__.py@@ -389,6 +389,7 @@ class ClaudeCodeAgentNode(ActionNode):connected_skill_names=connected_skills,connected_tools=tool_data,connected_memory=memory_data,+ execution_id=ctx.execution_id,allowed_credentials=params.allowed_credentials,max_parallel=params.max_parallel,)diff --git a/server/nodes/agent/claude_code_agent/_pool.py b/server/nodes/agent/claude_code_agent/_pool.pyindex 793290bc..2a8d83d6 100644--- a/server/nodes/agent/claude_code_agent/_pool.py+++ b/server/nodes/agent/claude_code_agent/_pool.py@@ -147,6 +147,11 @@ class PooledClaudeSession:# watcher picks up add/remove events without respawning.materialised_skills: frozenset = field(default_factory=frozenset)last_used_at: float = field(default_factory=time.monotonic)+ # Lease held from ``acquire`` through context rebinding, ``send_turn``,+ # and ``release``. The narrower ``lock`` below only protected stdin;+ # without this lease a second batch could replace the MCP BatchContext+ # after the first batch acquired the session but before its turn ran.+ turn_lease: asyncio.Lock = field(default_factory=asyncio.Lock)# Per-session lock; held throughout a turn. The reaper consults# ``lock.locked()`` to skip in-flight sessions.lock: asyncio.Lock = field(default_factory=asyncio.Lock)@@ -246,141 +251,190 @@ class ClaudeSessionPool:Caller MUST call :meth:`release` so the reaper can mark idle."""- async with self._pool_lock:- existing = self._pool.get(memory_node_id)- crashed_uuid = ""- if existing is not None and existing.process.returncode is not None:- logger.info(- "[ClaudeSessionPool] dropping dead session " "memory_node=%s pid=%s exit=%s — will respawn",- memory_node_id,- existing.process.pid,- existing.process.returncode,- )- crashed_uuid = existing.current_session_uuid- await self._terminate_locked(memory_node_id, reason="crashed")- existing = None-- if existing is not None:- existing.last_used_at = time.monotonic()- # Rebind the persistent :class:`BatchContext` bound to- # the warm subprocess's argv-baked bearer token to the- # surface the current batch declares. Without this the- # per-handler scope check in- # ``workflow_tools._build_handler`` still sees the- # stale ``connected_tools`` from the spawn batch and- # lets disconnected tools fire (and FastMCP refcounts- # leak by 1 per batch indefinitely).- #- # Service.py just issued ``mcp_bearer_token`` (T_new)- # for this batch and called ``register_batch`` on it.- # We transplant T_new's BatchContext values onto- # T_warm (the argv-baked one), then drop T_new — it- # would otherwise sit orphaned in ``_active_tokens``- # until app shutdown.- if existing.batch_token and mcp_bearer_token and mcp_bearer_token != existing.batch_token:- from services.cli_agent.mcp_server import (- lookup_batch,- rebind_batch,- unregister_batch,+ while True:+ # Capture or create the entry under the global map lock, but do+ # not wait for a busy per-memory lease here: a queued turn for M1+ # must never prevent an unrelated M2 from being acquired.+ async with self._pool_lock:+ existing = self._pool.get(memory_node_id)+ crashed_uuid = ""+ if existing is not None and existing.process.returncode is not None:+ logger.info(+ "[ClaudeSessionPool] dropping dead session "+ "memory_node=%s pid=%s exit=%s — will respawn",+ memory_node_id,+ existing.process.pid,+ existing.process.returncode,)-- new_ctx = lookup_batch(mcp_bearer_token)- if new_ctx is not None:- rebind_batch(- existing.batch_token,- connected_tools=new_ctx.connected_tools,- connected_skill_names=new_ctx.connected_skill_names,- allowed_credentials=new_ctx.allowed_credentials,- workspace_dir=new_ctx.workspace_dir,- )- unregister_batch(mcp_bearer_token)- # Live skill-tree update — claude live-watches- # ``<workspace_dir>/.claude/skills/`` so add/remove- # takes effect within the current session per the- # skills spec's "Live change detection" rule- # (code.claude.com/docs/en/skills#live-change-detection).- # Same diff semantics as the MCP rebind above:- # ``apply_skill_diff`` walks (added, removed) against- # ``existing.materialised_skills`` and only touches- # the delta on disk.- if existing.workspace_dir is not None:- from ._skills import materialise_skills-- new_set = frozenset(connected_skill_names or [])- added, removed = await materialise_skills(- existing.workspace_dir,- new_set,- previous_skill_names=existing.materialised_skills,- log_label=f"pool {memory_node_id}",+ crashed_uuid = existing.current_session_uuid+ await self._terminate_locked(memory_node_id, reason="crashed")+ existing = None++ if existing is None:+ if len(self._pool) >= self._max_size:+ await self._evict_lru_locked()++ # When recovering from a crash, splice the captured UUID+ # into the spec so the spawn emits ``--resume <UUID>``.+ if crashed_uuid and not spec.resume_session_id:+ spec.resume_session_id = crashed_uuid+ spec.continue_session = False++ session = await self._spawn(+ memory_node_id=memory_node_id,+ spec=spec,+ cwd=cwd,+ env=env,+ defaults=defaults,+ mcp_endpoint_url=mcp_endpoint_url,+ mcp_bearer_token=mcp_bearer_token,+ connected_tool_names=connected_tool_names,+ connected_skill_names=connected_skill_names,+ workspace_dir=workspace_dir,)- if added or removed:+ if mcp_bearer_token:+ session.batch_token = mcp_bearer_token+ session.workspace_dir = workspace_dir+ session.materialised_skills = frozenset(+ connected_skill_names or []+ )+ if crashed_uuid:+ session.current_session_uuid = crashed_uuid+ self._pool[memory_node_id] = session+ await session.turn_lease.acquire() # new; never contended+ try:logger.info(- "[ClaudeSessionPool] skill diff " "memory_node=%s +%d -%d (now %d wired)",+ "[ClaudeSessionPool] spawned new session memory_node=%s pid=%s",memory_node_id,- added,- removed,- len(new_set),+ session.process.pid,)- existing.materialised_skills = new_set- logger.info(- "[ClaudeSessionPool] warm reuse memory_node=%s pid=%s uuid=%s",- memory_node_id,- existing.process.pid,- existing.current_session_uuid or "(unresolved)",- )- return existing-- if len(self._pool) >= self._max_size:- await self._evict_lru_locked()-- # When recovering from a crash, splice the captured UUID- # into the spec so the spawn emits ``--resume <UUID>``.- if crashed_uuid and not spec.resume_session_id:- spec.resume_session_id = crashed_uuid- spec.continue_session = False+ await self._emit_event(+ "spawned",+ memory_node_id=memory_node_id,+ workflow_id=workflow_id,+ session_uuid=session.current_session_uuid,+ pid=session.process.pid,+ )+ except BaseException:+ if session.turn_lease.locked():+ session.turn_lease.release()+ await self._terminate_locked(+ memory_node_id,+ reason="acquire_failed",+ )+ raise+ return session- session = await self._spawn(+ # ``existing`` was captured while holding the map lock. Wait only+ # on its own lease, then verify it was not evicted/replaced while+ # we waited before rebinding any context.+ await existing.turn_lease.acquire()+ try:+ async with self._pool_lock:+ still_current = (+ self._pool.get(memory_node_id) is existing+ and existing.process.returncode is None+ )+ except BaseException:+ # Cancellation can land while waiting for the map lock after+ # this turn already acquired its per-memory lease.+ if existing.turn_lease.locked():+ existing.turn_lease.release()+ raise+ if not still_current:+ existing.turn_lease.release()+ continue++ return await self._prepare_warm_reuse(+ existing,memory_node_id=memory_node_id,- spec=spec,- cwd=cwd,- env=env,- defaults=defaults,- mcp_endpoint_url=mcp_endpoint_url,mcp_bearer_token=mcp_bearer_token,- connected_tool_names=connected_tool_names,connected_skill_names=connected_skill_names,- workspace_dir=workspace_dir,)- # Track the spawn-time bearer token so subsequent batches can- # rebind the persistent BatchContext in place (warm-reuse- # path above) and ``_terminate_locked`` can drain FastMCP- # tool refcounts on death.- if mcp_bearer_token:- session.batch_token = mcp_bearer_token- # Track the workspace + initial skill set so warm-reuse- # paths know where to materialise live updates and what- # the prior set was for the diff.- session.workspace_dir = workspace_dir- session.materialised_skills = frozenset(connected_skill_names or [])- if crashed_uuid:- # The new subprocess is resuming the prior session; pre-seed- # the UUID so the first turn's emitted CloudEvents carry- # the right id even before the result event lands.- session.current_session_uuid = crashed_uuid- self._pool[memory_node_id] = session++ async def _prepare_warm_reuse(+ self,+ existing: PooledClaudeSession,+ *,+ memory_node_id: str,+ mcp_bearer_token: Optional[str],+ connected_skill_names: Optional[List[str]],+ ) -> PooledClaudeSession:+ """Rebind one already-leased warm process for the next turn."""+ try:+ existing.last_used_at = time.monotonic()+ if (+ existing.batch_token+ and mcp_bearer_token+ and mcp_bearer_token != existing.batch_token+ ):+ from services.cli_agent.mcp_server import (+ lookup_batch,+ rebind_batch,+ unregister_batch,+ )++ new_ctx = lookup_batch(mcp_bearer_token)+ if new_ctx is not None:+ rebind_batch(+ existing.batch_token,+ workflow_id=new_ctx.workflow_id,+ node_id=new_ctx.node_id,+ execution_id=new_ctx.execution_id,+ broadcaster=new_ctx.broadcaster,+ connected_tools=new_ctx.connected_tools,+ connected_skill_names=new_ctx.connected_skill_names,+ allowed_credentials=new_ctx.allowed_credentials,+ workspace_dir=new_ctx.workspace_dir,+ )+ unregister_batch(mcp_bearer_token)++ if existing.workspace_dir is not None:+ from ._skills import materialise_skills++ new_set = frozenset(connected_skill_names or [])+ added, removed = await materialise_skills(+ existing.workspace_dir,+ new_set,+ previous_skill_names=existing.materialised_skills,+ log_label=f"pool {memory_node_id}",+ )+ if added or removed:+ logger.info(+ "[ClaudeSessionPool] skill diff "+ "memory_node=%s +%d -%d (now %d wired)",+ memory_node_id,+ added,+ removed,+ len(new_set),+ )+ existing.materialised_skills = new_set+logger.info(- "[ClaudeSessionPool] spawned new session memory_node=%s pid=%s",+ "[ClaudeSessionPool] warm reuse memory_node=%s pid=%s uuid=%s",memory_node_id,- session.process.pid,- )- await self._emit_event(- "spawned",- memory_node_id=memory_node_id,- workflow_id=workflow_id,- session_uuid=session.current_session_uuid,- pid=session.process.pid,+ existing.process.pid,+ existing.current_session_uuid or "(unresolved)",)- return session+ return existing+ except BaseException:+ # ``acquire`` did not return a session, so its caller has no+ # object it can pass to ``release``. Also discard the fresh token+ # allocated for a failed warm rebind; service.py deliberately+ # leaves pool-path token ownership to this class.+ if (+ mcp_bearer_token+ and mcp_bearer_token != existing.batch_token+ ):+ try:+ from services.cli_agent.mcp_server import unregister_batch++ unregister_batch(mcp_bearer_token)+ except Exception:+ pass+ if existing.turn_lease.locked():+ existing.turn_lease.release()+ raiseasync def clear(self,@@ -413,8 +467,10 @@ class ClaudeSessionPool:return ""async def release(self, session: PooledClaudeSession) -> None:- """Update ``last_used_at`` so the reaper measures from now."""+ """Release the context+turn lease and start the idle clock."""session.last_used_at = time.monotonic()+ if session.turn_lease.locked():+ session.turn_lease.release()async def terminate(self, memory_node_id: str) -> None:"""Force-drop a specific pooled session."""@@ -889,7 +945,11 @@ class ClaudeSessionPool:async def _evict_lru_locked(self) -> None:"""Caller holds ``self._pool_lock``. Evicts the least-recently-used non-in-flight entry."""- idle_entries = [(key, sess) for key, sess in self._pool.items() if not sess.lock.locked()]+ idle_entries = [+ (key, sess)+ for key, sess in self._pool.items()+ if not sess.lock.locked() and not sess.turn_lease.locked()+ ]if not idle_entries:logger.info("[ClaudeSessionPool] all %d entries in-flight; pool over cap",@@ -992,7 +1052,11 @@ class ClaudeSessionPool:now = time.monotonic()async with self._pool_lock:expired_keys = [- key for key, sess in self._pool.items() if not sess.lock.locked() and now - sess.last_used_at >= self._idle_ttl+ key+ for key, sess in self._pool.items()+ if not sess.lock.locked()+ and not sess.turn_lease.locked()+ and now - sess.last_used_at >= self._idle_ttl]for key in expired_keys:logger.info(diff --git a/server/nodes/agent/vertex_managed_agent/__init__.py b/server/nodes/agent/vertex_managed_agent/__init__.pyindex 0c9a4fe5..67959519 100644--- a/server/nodes/agent/vertex_managed_agent/__init__.py+++ b/server/nodes/agent/vertex_managed_agent/__init__.py@@ -328,14 +328,25 @@ class VertexManagedAgentNode(ActionNode):prev_interaction_idand cause is not Noneand (is_expired_environment_error(cause) or is_precondition_failure(cause))- ):+ ):logger.warning("[Vertex Agent] unresumable interaction chain for %s (%s) — retrying fresh",node_id,str(cause).replace("\n", " ")[:120],)if memory_node_id:- await self._save_chain_ids(database, memory_node_id, None, None)+ await self._save_chain_ids(+ database,+ memory_node_id,+ None,+ None,+ mutation_id=(+ f"vertex-chain-reset:{ctx.execution_id}:"+ f"{node_id}:{memory_node_id}"+ if ctx.execution_id+ else None+ ),+ )prev_interaction_id = Noneenvironment = "remote"interaction = await run_turn(input=prompt, environment="remote")@@ -441,6 +452,11 @@ class VertexManagedAgentNode(ActionNode):human=prompt,assistant=response_text,window_size=int((memory_data or {}).get("window_size") or 10),+ mutation_id=(+ f"vertex-memory:{ctx.execution_id}:{node_id}:{memory_node_id}"+ if ctx.execution_id+ else None+ ),)# ---- usage bookkeeping (tokens billed by Google; recorded for stats)@@ -986,28 +1002,40 @@ class VertexManagedAgentNode(ActionNode):human: Optional[str] = None,assistant: Optional[str] = None,window_size: int = 10,+ mutation_id: Optional[str] = None,) -> None:"""Persist chain ids (and optionally the turn) onto simpleMemory."""- from services.memory.markdown import (- append_to_memory_markdown,- trim_markdown_window,+ from services.memory.runtime import (+ append_memory_turns_atomic,+ update_memory_parameters_atomic,)- params = await database.get_node_parameters(memory_node_id) or {}+ updates: Dict[str, Any] = {}+ removals: List[str] = []if interaction_id:- params["vertex_interaction_id"] = interaction_id+ updates["vertex_interaction_id"] = interaction_idelse:- params.pop("vertex_interaction_id", None)+ removals.append("vertex_interaction_id")if environment_id:- params["vertex_environment_id"] = environment_id+ updates["vertex_environment_id"] = environment_idelse:- params.pop("vertex_environment_id", None)+ removals.append("vertex_environment_id")if human and assistant:- content = params.get("memory_content") or ""- content = append_to_memory_markdown(content, "human", human)- content = append_to_memory_markdown(content, "assistant", assistant)- content, _removed = trim_markdown_window(content, window_size)- params["memory_content"] = content-- await database.save_node_parameters(memory_node_id, params)+ await append_memory_turns_atomic(+ database,+ memory_node_id,+ [("human", human), ("assistant", assistant)],+ window_size=window_size,+ mutation_id=mutation_id,+ parameter_updates=updates,+ remove_parameters=removals,+ )+ else:+ await update_memory_parameters_atomic(+ database,+ memory_node_id,+ parameter_updates=updates,+ remove_parameters=removals,+ mutation_id=mutation_id,+ )diff --git a/server/nodes/proxy/proxy_config/__init__.py b/server/nodes/proxy/proxy_config/__init__.pyindex f62892ba..609d75c2 100644--- a/server/nodes/proxy/proxy_config/__init__.py+++ b/server/nodes/proxy/proxy_config/__init__.py@@ -320,7 +320,7 @@ async def _add_routing_rule(p: Dict[str, Any], proxy_svc) -> Dict[str, Any]:})if proxy_svc:- await proxy_svc.reload_providers()+ await proxy_svc.reload_routing_rules()return {"success": True, "domain_pattern": domain_pattern}@@ -333,7 +333,7 @@ async def _remove_routing_rule(p: Dict[str, Any], proxy_svc) -> Dict[str, Any]:db = get_database()await db.delete_proxy_routing_rule(int(rule_id))if proxy_svc:- await proxy_svc.reload_providers()+ await proxy_svc.reload_routing_rules()return {"success": True, "rule_id": rule_id}diff --git a/server/nodes/tool/agent_builder/__init__.py b/server/nodes/tool/agent_builder/__init__.pyindex 8cab5c38..cf6ca267 100644--- a/server/nodes/tool/agent_builder/__init__.py+++ b/server/nodes/tool/agent_builder/__init__.py@@ -30,6 +30,9 @@ to call something that isn't there yet.from __future__ import annotations+import hashlib+import inspect+import jsonimport uuidfrom pathlib import Pathfrom typing import Any, Dict, List, Literal, Optional@@ -76,7 +79,118 @@ _CREATE_WORKFLOW_ENABLED = False# -----------------------------------------------------------------------------async def _persist_canvas_mutation(workflow_id: Optional[str], ops: List[Dict[str, Any]]) -> None:+def _apply_canvas_ops(+ data: Dict[str, Any],+ ops: List[Dict[str, Any]],+) -> tuple[Dict[str, Any], Dict[str, Any]]:+ """Apply an agentBuilder batch to one current workflow snapshot.++ Minted node IDs and exact edge identities are de-duplicated so retrying+ the same batch is harmless. Distinct batches retain their distinct minted+ nodes even when they happen to add the same type concurrently.+ """+ data = dict(data or {})+ nodes = [dict(node) for node in (data.get("nodes") or [])]+ edges = [dict(edge) for edge in (data.get("edges") or [])]+ ref_to_id: Dict[str, str] = {}+ changed = False++ def _resolve(ref: Any) -> Optional[str]:+ if isinstance(ref, str):+ return ref+ if isinstance(ref, dict) and "client_ref" in ref:+ return ref_to_id.get(ref["client_ref"])+ return None++ for op in ops:+ op_type = op.get("type")+ if op_type == "add_node":+ node_type = op.get("node_type") or ""+ if not node_type:+ continue+ new_id = op.get("minted_id") or op.get("client_ref") or ""+ if not new_id:+ continue+ client_ref = op.get("client_ref") or ""++ if client_ref:+ ref_to_id[client_ref] = new_id+ if any(node.get("id") == new_id for node in nodes):+ continue+ params = op.get("parameters") or {}+ nodes.append(+ {+ "id": new_id,+ "type": node_type,+ "position": (+ op.get("position")+ if isinstance(op.get("position"), dict)+ and "x" in op["position"]+ else {"x": 0, "y": 0}+ ),+ "data": {+ "label": op.get("label") or params.get("label") or node_type,+ "parameters": dict(params),+ },+ }+ )+ changed = True+ elif op_type == "add_edge":+ source = _resolve(op.get("source"))+ target = _resolve(op.get("target"))+ if not source or not target:+ continue+ identity = (+ source,+ target,+ op.get("source_handle"),+ op.get("target_handle"),+ )+ if any(+ (+ edge.get("source"),+ edge.get("target"),+ edge.get("sourceHandle"),+ edge.get("targetHandle"),+ )+ == identity+ for edge in edges+ ):+ continue+ edges.append(+ {+ "id": f"e-{source}-{target}",+ "source": source,+ "target": target,+ "sourceHandle": op.get("source_handle"),+ "targetHandle": op.get("target_handle"),+ }+ )+ changed = True+ elif op_type == "set_node_parameters":+ node_id = op.get("node_id")+ merge = op.get("parameters") or {}+ for node in nodes:+ if node.get("id") != node_id:+ continue+ node_data = dict(node.get("data") or {})+ existing = dict(node_data.get("parameters") or {})+ merged = {**existing, **merge}+ if merged != existing:+ node_data["parameters"] = merged+ node["data"] = node_data+ changed = True+ break++ data["nodes"] = nodes+ data["edges"] = edges+ return data, {"changed": changed, "nodes": len(nodes), "edges": len(edges)}+++async def _persist_canvas_mutation(+ workflow_id: Optional[str],+ ops: List[Dict[str, Any]],+) -> Optional[bool]:"""Apply the three op types agentBuilder emits to the persisted``workflow.data`` via the existing ``database.get_workflow`` +``database.save_workflow`` round-trip.@@ -91,85 +205,60 @@ async def _persist_canvas_mutation(workflow_id: Optional[str], ops: List[Dict[stops for FE alignment) so the persisted row matches the React Flowstate exactly — no id divergence on reload."""- if not ops or not workflow_id:- return+ if not ops:+ return False+ if not workflow_id:+ return Nonetry:from services.plugin.deps import get_databasedatabase = get_database()+ atomic_mutate = getattr(database, "mutate_workflow_data_atomic", None)+ if atomic_mutate is not None and inspect.iscoroutinefunction(atomic_mutate):+ canonical_ops = json.dumps(+ ops,+ sort_keys=True,+ separators=(",", ":"),+ default=str,+ )+ mutation_id = (+ f"agent-builder:{workflow_id}:"+ f"{hashlib.sha256(canonical_ops.encode('utf-8')).hexdigest()}"+ )+ workflow, metadata, applied = await atomic_mutate(+ workflow_id,+ lambda current: _apply_canvas_ops(current, ops),+ mutation_id=mutation_id,+ operation="agent_builder_ops",+ )+ if workflow is None or not metadata.get("found", True):+ logger.debug(+ "[agentBuilder] persist skipped: workflow %s not found",+ workflow_id,+ )+ return None+ logger.info(+ "[agentBuilder] transactionally persisted %s ops to workflow %s "+ "(nodes=%d edges=%d applied=%s)",+ [op.get("type") for op in ops],+ workflow_id,+ metadata.get("nodes", 0),+ metadata.get("edges", 0),+ applied,+ )+ return bool(applied and metadata.get("changed"))++ # Compatibility path for existing isolated test doubles.workflow = await database.get_workflow(workflow_id)if workflow is None:logger.debug("[agentBuilder] persist skipped: workflow %s not found", workflow_id)- return-- data = dict(workflow.data or {})- nodes = list(data.get("nodes") or [])- edges = list(data.get("edges") or [])- ref_to_id: Dict[str, str] = {}-- def _resolve(ref: Any) -> Optional[str]:- if isinstance(ref, str):- return ref- if isinstance(ref, dict) and "client_ref" in ref:- return ref_to_id.get(ref["client_ref"])return None- for op in ops:- op_type = op.get("type")- if op_type == "add_node":- node_type = op.get("node_type") or ""- if not node_type:- continue- # ``minted_id`` is set by add_tool / add_skill / add_subagent- # so the FE applier adopts the same id. Reuse it for the- # persisted row to keep BE/FE views aligned.- new_id = op.get("minted_id") or op.get("client_ref") or ""- if not new_id:- continue- client_ref = op.get("client_ref") or ""- if client_ref:- ref_to_id[client_ref] = new_id- params = op.get("parameters") or {}- nodes.append(- {- "id": new_id,- "type": node_type,- "position": op.get("position") if isinstance(op.get("position"), dict) and "x" in op["position"] else {"x": 0, "y": 0},- "data": {- "label": op.get("label") or params.get("label") or node_type,- "parameters": dict(params),- },- }- )- elif op_type == "add_edge":- source = _resolve(op.get("source"))- target = _resolve(op.get("target"))- if not source or not target:- continue- edges.append(- {- "id": f"e-{source}-{target}",- "source": source,- "target": target,- "sourceHandle": op.get("source_handle"),- "targetHandle": op.get("target_handle"),- }- )- elif op_type == "set_node_parameters":- node_id = op.get("node_id")- merge = op.get("parameters") or {}- for n in nodes:- if n.get("id") != node_id:- continue- node_data = dict(n.get("data") or {})- existing = dict(node_data.get("parameters") or {})- existing.update(merge)- node_data["parameters"] = existing- n["data"] = node_data- break-- data["nodes"] = nodes- data["edges"] = edges+ data, metadata = _apply_canvas_ops(dict(workflow.data or {}), ops)+ nodes = data.get("nodes") or []+ edges = data.get("edges") or []+ if not metadata.get("changed"):+ return Falseawait database.save_workflow(workflow_id=workflow_id,name=workflow.name,@@ -184,10 +273,12 @@ async def _persist_canvas_mutation(workflow_id: Optional[str], ops: List[Dict[stlen(nodes),len(edges),)+ return Trueexcept Exception as exc:logger.warning("[agentBuilder] persist failed for workflow %s: %s", workflow_id, exc, exc_info=True)+ return Noneasync def _broadcast(workflow_id: Optional[str], caller_id: str, ops: List[Dict[str, Any]]) -> None:@@ -203,7 +294,12 @@ async def _broadcast(workflow_id: Optional[str], caller_id: str, ops: List[Dict["""if not ops:return- await _persist_canvas_mutation(workflow_id, ops)+ persisted = await _persist_canvas_mutation(workflow_id, ops)+ # ``None`` means persistence was unavailable/missing and is explicitly+ # best-effort: the live canvas still needs the operation event. ``False``+ # is a known idempotent no-op and should not be rebroadcast.+ if persisted is False:+ returntry:from ._events import broadcast_workflow_opsdiff --git a/server/services/agent_team.py b/server/services/agent_team.pyindex 7b7c3cfa..b13a882c 100644--- a/server/services/agent_team.py+++ b/server/services/agent_team.py@@ -130,11 +130,14 @@ class AgentTeamService:async def claim_task(self, team_id: str, task_id: str, agent_node_id: str) -> Optional[Dict[str, Any]]:"""Claim a task for an agent."""- # Update member status to working- await self.database.update_member_status(team_id, agent_node_id, "working")-task = await self.database.claim_task(task_id, agent_node_id)+ # Only the winner of the conditional database claim becomes working.+ # Updating first left every losing parallel contender stuck in the+ # working state even though exactly one owned the task.+ if task:+ await self.database.update_member_status(team_id, agent_node_id, "working")+if task and self.broadcaster:await self.broadcaster.broadcast_team_event(team_id, "task_claimed", {**task, "claimed_by": agent_node_id})diff --git a/server/services/ai.py b/server/services/ai.pyindex 4490db2d..264211c9 100644--- a/server/services/ai.py+++ b/server/services/ai.py@@ -41,6 +41,7 @@ import json# those typed errors cleanly through ``BaseNode.execute()``.import openaifrom services.plugin import NodeUserError+from services.tool_identity import DuplicateToolNameError, ensure_unique_tool_namesif TYPE_CHECKING:from langchain_openai import ChatOpenAI # noqa: F401@@ -851,6 +852,13 @@ async def _run_agent_loop(added = await rebind_from_operations(result["operations"])if added:iteration_new_tools.extend(added)+ except DuplicateToolNameError as exc:+ # Reject the whole hot-rebind batch before mutating the+ # bound surface. Return the structured conflict to the+ # model so it can rename/rebuild the ambiguous tool.+ logger.warning("[Agent loop] rejected ambiguous tool rebind: %s", exc)+ result = dict(result)+ result["rebind_error"] = exc.as_dict()except Exception as exc: # noqa: BLE001 — defensive; rebind is opt-inlogger.warning("[Agent loop] rebind_from_operations raised: %s", exc, exc_info=True@@ -1731,6 +1739,8 @@ class AIService:# Build tools if providedtools = []tool_configs = {}+ tool_identities: List[Dict[str, str]] = []+ tool_bindings: List[tuple[Any, Dict[str, Any]]] = []if tool_data:await broadcast_status("building_tools", {"message": f"Building {len(tool_data)} tool(s)...", "tool_count": len(tool_data)})@@ -1739,13 +1749,20 @@ class AIService:tool, config = await self._build_tool_from_node(tool_info)if tool:tools.append(tool)- tool_configs[tool.name] = config+ tool_bindings.append((tool, config))+ tool_identities.append(+ {+ "name": tool.name,+ "node_id": str(config.get("node_id") or tool_info.get("node_id") or ""),+ "label": str(config.get("label") or tool_info.get("label") or tool_info.get("node_type") or "tool"),+ }+ )logger.debug(f"[Agent] Registered tool: name={tool.name}, node_id={config.get('node_id')}")logger.debug(f"[Agent] Built {len(tools)} tools")# Auto-inject check_delegated_tasks tool when delegation tools present- if any(name.startswith("delegate_to_") for name in tool_configs):+ if any(tool.name.startswith("delegate_to_") for tool in tools):check_info = {"node_type": "_builtin_check_delegated_tasks","node_id": f"{node_id}_check_tasks",@@ -1755,11 +1772,18 @@ class AIService:check_tool, check_config = await self._build_tool_from_node(check_info)if check_tool:tools.append(check_tool)- tool_configs[check_tool.name] = check_config+ tool_bindings.append((check_tool, check_config))+ tool_identities.append(+ {+ "name": check_tool.name,+ "node_id": str(check_config.get("node_id") or check_info["node_id"]),+ "label": str(check_config.get("label") or check_info["label"]),+ }+ )logger.debug("[Agent] Auto-injected check_delegated_tasks tool")# Add delegation guidance to system message- delegate_names = [n for n in tool_configs if n.startswith("delegate_to_")]+ delegate_names = [tool.name for tool in tools if tool.name.startswith("delegate_to_")]system_message += ("\n\n## Agent Delegation\n""When delegating to sub-agents, use 'task' for the mission directive "@@ -1767,6 +1791,12 @@ class AIService:f"Available agents: {', '.join(delegate_names)}")+ # Name-based dispatch is only safe after validating the full+ # surface. Build the lookup map after this check so a later+ # duplicate can never silently replace an earlier node.+ ensure_unique_tool_names(tool_identities)+ tool_configs.update({tool.name: config for tool, config in tool_bindings})+# Now that ``system_message`` is final (skill prompt + delegation# guidance both folded in), prepend it as the first message.# See the comment at the original ``initial_messages`` declaration@@ -1860,7 +1890,7 @@ class AIService:"""from services.node_registry import get_node_class- new_tools: List[Any] = []+ new_bindings: List[tuple[Any, Dict[str, Any]]] = []for op in operations:if op.get("type") != "add_node":continue@@ -1896,10 +1926,21 @@ class AIService:continueif tool is None:continue- new_tools.append(tool)- if tool_config:- tool_configs[tool.name] = tool_config- return new_tools+ new_bindings.append((tool, tool_config or tool_info))++ new_identities = [+ {+ "name": tool.name,+ "node_id": str(config.get("node_id") or ""),+ "label": str(config.get("label") or config.get("node_type") or "tool"),+ }+ for tool, config in new_bindings+ ]+ ensure_unique_tool_names([*tool_identities, *new_identities])+ for tool, tool_config in new_bindings:+ tool_configs[tool.name] = tool_config+ tool_identities.extend(new_identities)+ return [tool for tool, _ in new_bindings]# Broadcast: Building agentawait broadcast_status(@@ -2014,22 +2055,33 @@ class AIService:},)- # If compaction happened, use compacted summary as base+ from services.memory.runtime import append_memory_turns_atomic++ # Compaction may take long enough for another worker to append.+ # The atomic helper installs the summary only if the content it+ # compacted is still current; otherwise it preserves the newer+ # transcript and appends this exchange to it.+ compacted_summary = Noneif compaction_result and compaction_result.get("success") and compaction_result.get("summary"):- # Start fresh with compacted summary, then add current exchange- updated_content = compaction_result["summary"]- updated_content = _append_to_memory_markdown(updated_content, "human", prompt)- updated_content = _append_to_memory_markdown(updated_content, "ai", response_content)+ compacted_summary = compaction_result["summary"]logger.info("[Agent Memory] Using compacted summary as new base")- else:- # Normal flow: append to existing memory- updated_content = memory_data.get("memory_content", "# Conversation History\n\n*No messages yet.*\n")- updated_content = _append_to_memory_markdown(updated_content, "human", prompt)- updated_content = _append_to_memory_markdown(updated_content, "ai", response_content)- # Trim to window size, archive removed to vector DB- window_size = memory_data.get("window_size", 10)- updated_content, removed_texts = _trim_markdown_window(updated_content, window_size)+ memory_node_id = memory_data["node_id"]+ execution_id = (context or {}).get("execution_id")+ mutation_id = (+ f"ai-memory:{execution_id}:{node_id}:{memory_node_id}"+ if execution_id+ else None+ )+ _current_params, removed_texts, _applied = await append_memory_turns_atomic(+ self.database,+ memory_node_id,+ [("human", prompt), ("ai", response_content)],+ window_size=int(memory_data.get("window_size", 10)),+ mutation_id=mutation_id,+ replacement_content=compacted_summary,+ expected_content=memory_data.get("memory_content", ""),+ )# Store removed messages in long-term vector DBif removed_texts and memory_data.get("long_term_enabled"):@@ -2041,14 +2093,6 @@ class AIService:except Exception as e:logger.warning(f"[Agent Memory] Failed to archive to vector store: {e}")- # Save updated markdown to node parameters. Schema-- # canonical key is snake_case; drop any pre-migration- # camelCase mirror so the saved params stay clean.- memory_node_id = memory_data["node_id"]- current_params = await self.database.get_node_parameters(memory_node_id) or {}- current_params["memory_content"] = updated_content- current_params.pop("memoryContent", None)- await self.database.save_node_parameters(memory_node_id, current_params)logger.info(f"[Agent Memory] Saved markdown to memory node '{memory_node_id}'")result = {@@ -2082,6 +2126,17 @@ class AIService:"execution_time": time.time() - start_time,}+ except DuplicateToolNameError as e:+ logger.warning("[Agent] Duplicate tool surface rejected: %s", e)+ return {+ "success": False,+ "node_id": node_id,+ "node_type": "aiAgent",+ **e.as_dict(),+ "execution_time": time.time() - start_time,+ "timestamp": datetime.now().isoformat(),+ }+except openai.OpenAIError as e:# See execute_chat for the rationale — typed SDK errors are# user-correctable and re-raised as NodeUserError so BaseNode@@ -2167,6 +2222,8 @@ class AIService:# This supports ALL tool types: calculatorTool, currentTimeTool, duckduckgoSearch, androidTool, httpRequestall_tools = []tool_node_configs = {} # Map tool name to node config (same as AI Agent's tool_configs)+ tool_identities: List[Dict[str, str]] = []+ tool_bindings: List[tuple[Any, Dict[str, Any]]] = []if tool_data:await broadcast_status("building_tools", {"message": f"Building {len(tool_data)} tool(s)...", "tool_count": len(tool_data)})@@ -2175,7 +2232,14 @@ class AIService:tool, config = await self._build_tool_from_node(tool_info)if tool:all_tools.append(tool)- tool_node_configs[tool.name] = config+ tool_bindings.append((tool, config))+ tool_identities.append(+ {+ "name": tool.name,+ "node_id": str(config.get("node_id") or tool_info.get("node_id") or ""),+ "label": str(config.get("label") or tool_info.get("label") or tool_info.get("node_type") or "tool"),+ }+ )logger.debug(f"[ChatAgent] Built tool: {tool.name} (type={config.get('node_type')}, node_id={config.get('node_id')})")@@ -2183,7 +2247,7 @@ class AIService:logger.debug(f"[ChatAgent] Built {len(all_tools)} tools from tool_data")# Auto-inject check_delegated_tasks tool when delegation tools present- if any(name.startswith("delegate_to_") for name in tool_node_configs):+ if any(tool.name.startswith("delegate_to_") for tool in all_tools):check_info = {"node_type": "_builtin_check_delegated_tasks","node_id": f"{node_id}_check_tasks",@@ -2193,11 +2257,18 @@ class AIService:check_tool, check_config = await self._build_tool_from_node(check_info)if check_tool:all_tools.append(check_tool)- tool_node_configs[check_tool.name] = check_config+ tool_bindings.append((check_tool, check_config))+ tool_identities.append(+ {+ "name": check_tool.name,+ "node_id": str(check_config.get("node_id") or check_info["node_id"]),+ "label": str(check_config.get("label") or check_info["label"]),+ }+ )logger.debug("[ChatAgent] Auto-injected check_delegated_tasks tool")# Add delegation guidance to system message- delegate_names = [n for n in tool_node_configs if n.startswith("delegate_to_")]+ delegate_names = [tool.name for tool in all_tools if tool.name.startswith("delegate_to_")]system_message += ("\n\n## Agent Delegation\n""When delegating to sub-agents, use 'task' for the mission directive "@@ -2205,6 +2276,9 @@ class AIService:f"Available agents: {', '.join(delegate_names)}")+ ensure_unique_tool_names(tool_identities)+ tool_node_configs.update({tool.name: config for tool, config in tool_bindings})+logger.debug(f"[ChatAgent] Total tools available: {len(all_tools)}")# Debug: log all tool schemas to verify they're correctfor t in all_tools:@@ -2389,7 +2463,7 @@ class AIService:(``tool_configs`` vs ``tool_node_configs``)."""from services.node_registry import get_node_class- new_tools: List[Any] = []+ new_bindings: List[tuple[Any, Dict[str, Any]]] = []for op in operations:if op.get("type") != "add_node":continue@@ -2424,10 +2498,21 @@ class AIService:continueif tool is None:continue- new_tools.append(tool)- if tool_config:- tool_node_configs[tool.name] = tool_config- return new_tools+ new_bindings.append((tool, tool_config or tool_info))++ new_identities = [+ {+ "name": tool.name,+ "node_id": str(config.get("node_id") or ""),+ "label": str(config.get("label") or config.get("node_type") or "tool"),+ }+ for tool, config in new_bindings+ ]+ ensure_unique_tool_names([*tool_identities, *new_identities])+ for tool, tool_config in new_bindings:+ tool_node_configs[tool.name] = tool_config+ tool_identities.extend(new_identities)+ return [tool for tool, _ in new_bindings]# Run the agent loop. See ``execute_agent`` for the# rationale on ``recursion_limit`` + the truncation@@ -2513,21 +2598,29 @@ class AIService:"saving_memory", {"message": "Saving to conversation memory...", "session_id": session_id, "has_memory": True})- # If compaction happened, use compacted summary as base+ from services.memory.runtime import append_memory_turns_atomic++ compacted_summary = Noneif compaction_result and compaction_result.get("success") and compaction_result.get("summary"):- updated_content = compaction_result["summary"]- updated_content = _append_to_memory_markdown(updated_content, "human", prompt)- updated_content = _append_to_memory_markdown(updated_content, "ai", response_content)+ compacted_summary = compaction_result["summary"]logger.info("[ChatAgent Memory] Using compacted summary as new base")- else:- # Normal flow: append to existing memory- updated_content = memory_content or "# Conversation History\n\n*No messages yet.*\n"- updated_content = _append_to_memory_markdown(updated_content, "human", prompt)- updated_content = _append_to_memory_markdown(updated_content, "ai", response_content)- # Trim to window size, archive removed to vector DB- window_size = memory_data.get("window_size", 10)- updated_content, removed_texts = _trim_markdown_window(updated_content, window_size)+ memory_node_id = memory_data["node_id"]+ execution_id = (context or {}).get("execution_id")+ mutation_id = (+ f"chat-memory:{execution_id}:{node_id}:{memory_node_id}"+ if execution_id+ else None+ )+ _current_params, removed_texts, _applied = await append_memory_turns_atomic(+ self.database,+ memory_node_id,+ [("human", prompt), ("ai", response_content)],+ window_size=int(memory_data.get("window_size", 10)),+ mutation_id=mutation_id,+ replacement_content=compacted_summary,+ expected_content=memory_content,+ )# Store removed messages in long-term vector DBif removed_texts and memory_data.get("long_term_enabled"):@@ -2539,14 +2632,6 @@ class AIService:except Exception as e:logger.warning(f"[ChatAgent Memory] Failed to archive to vector store: {e}")- # Save updated markdown to node parameters. Schema-- # canonical key is snake_case; drop any pre-migration- # camelCase mirror so the saved params stay clean.- memory_node_id = memory_data["node_id"]- current_params = await self.database.get_node_parameters(memory_node_id) or {}- current_params["memory_content"] = updated_content- current_params.pop("memoryContent", None)- await self.database.save_node_parameters(memory_node_id, current_params)logger.info(f"[ChatAgent Memory] Saved markdown to memory node '{memory_node_id}'")# Determine agent type based on configuration@@ -2597,6 +2682,17 @@ class AIService:"execution_time": time.time() - start_time,}+ except DuplicateToolNameError as e:+ logger.warning("[ChatAgent] Duplicate tool surface rejected: %s", e)+ return {+ "success": False,+ "node_id": node_id,+ "node_type": "chatAgent",+ **e.as_dict(),+ "execution_time": time.time() - start_time,+ "timestamp": datetime.now().isoformat(),+ }+except openai.OpenAIError as e:# Typed SDK error — see execute_chat for rationale.log_api_call(logger, provider, model, "chat_agent", False, error=str(e))diff --git a/server/services/cli_agent/mcp_server.py b/server/services/cli_agent/mcp_server.pyindex 48e5d6de..b83e423f 100644--- a/server/services/cli_agent/mcp_server.py+++ b/server/services/cli_agent/mcp_server.py@@ -36,6 +36,7 @@ from starlette.requests import Requestfrom starlette.responses import JSONResponse, Responsefrom core.logging import get_logger+from services.tool_identity import ensure_unique_tool_nameslogger = get_logger(__name__)@@ -57,6 +58,7 @@ class BatchContext:workflow_id: strnode_id: strworkspace_dir: Path+ execution_id: Optional[str] = Noneconnected_skill_names: Set[str] = field(default_factory=set)allowed_credentials: Set[str] = field(default_factory=set)# Connected ``input-tools`` nodes (entries from@@ -73,6 +75,7 @@ class BatchContext:# Token -> BatchContext registry. Lives in-memory only; tokens never# touch disk or the credentials.db._active_tokens: Dict[str, BatchContext] = {}+_UNSET = object()def issue_token() -> str:@@ -80,6 +83,19 @@ def issue_token() -> str:return secrets.token_hex(32)+def _validate_connected_tool_names(connected_tools: List[Dict[str, Any]]) -> None:+ """Reject ambiguous node-type dispatch within one CLI agent surface."""+ ensure_unique_tool_names(+ {+ "name": tool["node_type"],+ "node_id": tool.get("node_id") or "",+ "label": tool.get("label") or tool["node_type"],+ }+ for tool in connected_tools+ if tool.get("node_type")+ )++def register_batch(token: str, ctx: BatchContext) -> None:"""Register a batch's auth token + expose its connected workflowtools on the FastMCP server. Idempotent on identical context."""@@ -88,6 +104,11 @@ def register_batch(token: str, ctx: BatchContext) -> None:if existing is not ctx:raise ValueError("Token collision in MCP server batch registry")return+ # Validate before either the bearer-token registry or FastMCP's global+ # tool registry changes. The MCP handler dispatches by node_type, so two+ # physical nodes of the same effective name would otherwise select the+ # first one silently.+ _validate_connected_tool_names(ctx.connected_tools)_active_tokens[token] = ctxtool_names = [t.get("node_type") for t in ctx.connected_tools]logger.info(@@ -125,6 +146,10 @@ def lookup_batch(token: str) -> Optional[BatchContext]:def rebind_batch(token: str,*,+ workflow_id: Optional[str] = None,+ node_id: Optional[str] = None,+ execution_id: Any = _UNSET,+ broadcaster: Any = _UNSET,connected_tools: Optional[List[Dict[str, Any]]] = None,connected_skill_names: Optional[Set[str]] = None,allowed_credentials: Optional[Set[str]] = None,@@ -154,6 +179,19 @@ def rebind_batch(ctx = _active_tokens.get(token)if ctx is None:return None+ # Rebinding is transactional from the caller's perspective: an invalid+ # tool surface must not partially refresh identity, credentials, or MCP+ # refcounts on the warm process's persistent context.+ if connected_tools is not None:+ _validate_connected_tool_names(connected_tools)+ if workflow_id is not None:+ ctx.workflow_id = workflow_id+ if node_id is not None:+ ctx.node_id = node_id+ if execution_id is not _UNSET:+ ctx.execution_id = execution_id+ if broadcaster is not _UNSET:+ ctx.broadcaster = broadcasterif connected_tools is not None:from services.cli_agent.workflow_tools import (expose_workflow_tools,diff --git a/server/services/cli_agent/service.py b/server/services/cli_agent/service.pyindex 55c42c2b..7c311a7e 100644--- a/server/services/cli_agent/service.py+++ b/server/services/cli_agent/service.py@@ -78,6 +78,7 @@ class AICliService:connected_skill_names: Optional[List[str]] = None,connected_tools: Optional[List[Dict[str, Any]]] = None,connected_memory: Optional[Dict[str, Any]] = None,+ execution_id: Optional[str] = None,allowed_credentials: Optional[List[str]] = None,max_parallel: int = DEFAULT_MAX_PARALLEL,mcp_port: Optional[int] = None,@@ -172,6 +173,7 @@ class AICliService:workflow_id=workflow_id,node_id=node_id,workspace_dir=Path(workspace_dir).resolve(),+ execution_id=execution_id,connected_skill_names=set(connected_skill_names or []),allowed_credentials=set(allowed_credentials or []),connected_tools=list(connected_tools or []),@@ -318,6 +320,12 @@ class AICliService:connected_memory,results,broadcaster=broadcaster,+ mutation_id=(+ f"cli-memory:{execution_id}:{node_id}:"+ f"{connected_memory.get('node_id', '')}"+ if execution_id+ else None+ ),)except Exception as exc: # pragma: no cover — best-effortlogger.warning(@@ -461,19 +469,28 @@ class AICliService:env["OPENCOMPANY_PARENT_RUN_ID"] = parent_run_idenv["MACHINA_PARENT_RUN_ID"] = parent_run_id # legacy child-process contract- pooled = await pool.acquire(- memory_node_id,- spec=task,- cwd=cwd,- env=env,- defaults=defaults,- mcp_endpoint_url=mcp_endpoint_url,- mcp_bearer_token=mcp_bearer_token,- connected_tool_names=tool_names,- connected_skill_names=connected_skill_names,- workspace_dir=workspace_dir,- workflow_id=workflow_id,- )+ try:+ pooled = await pool.acquire(+ memory_node_id,+ spec=task,+ cwd=cwd,+ env=env,+ defaults=defaults,+ mcp_endpoint_url=mcp_endpoint_url,+ mcp_bearer_token=mcp_bearer_token,+ connected_tool_names=tool_names,+ connected_skill_names=connected_skill_names,+ workspace_dir=workspace_dir,+ workflow_id=workflow_id,+ )+ except BaseException:+ # A cold spawn can fail before the pool adopts the token. The+ # run_batch pool branch intentionally does not unregister in its+ # outer finally, so close this otherwise-orphaned context here.+ from services.cli_agent.mcp_server import unregister_batch++ unregister_batch(mcp_bearer_token)+ raisetry:return await pool.send_turn(pooled,@@ -504,8 +521,13 @@ class AICliService:prior = params.get("last_session_id")if not prior:return # already cleared- params["last_session_id"] = None- await db.save_node_parameters(memory_node_id, params)+ from services.memory.runtime import update_memory_parameters_atomic++ await update_memory_parameters_atomic(+ db,+ memory_node_id,+ parameter_updates={"last_session_id": None},+ )logger.warning("[CC-Agent _persist_memory] cleared stale last_session_id=%s ""from memory_node=%s; next run will spawn a fresh claude "@@ -519,6 +541,7 @@ class AICliService:connected_memory: Dict[str, Any],results: List[SessionResult],broadcaster: Any = None,+ mutation_id: Optional[str] = None,) -> None:"""Append each successful run's user prompt + assistant responseto ``simpleMemory.memory_content`` (markdown). Mirrors aiAgent /@@ -554,38 +577,37 @@ class AICliService:await AICliService._clear_stale_session_id(connected_memory)return- from services.memory import (- append_to_memory_markdown,- trim_markdown_window,- )+ from services.memory.runtime import append_memory_turns_atomicfrom services.plugin.deps import get_databasedb = get_database()memory_node_id = connected_memory["node_id"]- params = await db.get_node_parameters(memory_node_id) or {}# 1. Persist claude's returned session_id from the most recent# successful run. Drives `--resume <UUID>` on the next spawn so# claude finds and continues its own JSONL transcript on disk.last_run = next((r for r in reversed(successful) if r.session_id), None)- if last_run is not None:- params["last_session_id"] = last_run.session_id-- # 2. Update the user-visible markdown mirror.- content = params.get("memory_content") or ("# Conversation History\n\n*No messages yet.*\n")- for r in successful:- content = append_to_memory_markdown(content, "human", r.prompt)- content = append_to_memory_markdown(- content,- "ai",- r.response or "",- )-window = int(connected_memory.get("window_size") or 100)- content, removed_texts = trim_markdown_window(content, window)- params["memory_content"] = content-- await db.save_node_parameters(memory_node_id, params)+ params, removed_texts, _applied = await append_memory_turns_atomic(+ db,+ memory_node_id,+ [+ turn+ for result in successful+ for turn in (+ ("human", result.prompt),+ ("ai", result.response or ""),+ )+ ],+ window_size=window,+ mutation_id=mutation_id,+ parameter_updates=(+ {"last_session_id": last_run.session_id}+ if last_run is not None+ else None+ ),+ )+ content = params.get("memory_content") or ""logger.info("[CC-Agent _persist_memory] saved memory_node=%s ""last_session_id=%s appended_turns=%d archived_blocks=%d "diff --git a/server/services/cli_agent/workflow_tools.py b/server/services/cli_agent/workflow_tools.pyindex 3f6e7e3b..49977ee9 100644--- a/server/services/cli_agent/workflow_tools.py+++ b/server/services/cli_agent/workflow_tools.py@@ -210,6 +210,9 @@ def _build_handler(node_type: str, params_cls: type):"node_type": node_type,"node_id": entry.get("node_id"),"workflow_id": ctx.workflow_id,+ "execution_id": ctx.execution_id,+ "workspace_dir": str(ctx.workspace_dir),+ "parent_node_id": ctx.node_id,"label": entry.get("label") or node_type,"parameters": dict(entry.get("parameters") or {}),}diff --git a/server/services/memory/__init__.py b/server/services/memory/__init__.pyindex 56808193..69759ad9 100644--- a/server/services/memory/__init__.py+++ b/server/services/memory/__init__.py@@ -32,6 +32,10 @@ from services.memory.jsonl import (parse_jsonl,trim_window,)+from services.memory.runtime import (+ append_memory_turns_atomic,+ update_memory_parameters_atomic,+)__all__ = [# Markdown helpers — used by every agent bridge (aiAgent /@@ -49,4 +53,7 @@ __all__ = ["parse_jsonl","append_message","trim_window",+ # Transactional runtime writes.+ "append_memory_turns_atomic",+ "update_memory_parameters_atomic",]diff --git a/server/services/memory/runtime.py b/server/services/memory/runtime.pynew file mode 100644index 00000000..fd1d414a--- /dev/null+++ b/server/services/memory/runtime.py@@ -0,0 +1,136 @@+"""Atomic persistence helpers for runtime conversation memory.++All agent bridges share the same ``simpleMemory`` JSON row. Appending by+reading the row and later replacing it loses turns whenever two agents finish+at the same time. This module keeps the markdown transformation inside the+database's reserved write transaction and optionally records a durable+idempotency key for workflow/activity retries.+"""++from __future__ import annotations++import inspect+from typing import Any, Dict, Iterable, Optional, Sequence, Tuple++from services.memory.markdown import (+ append_to_memory_markdown,+ trim_markdown_window,+)++MemoryTurn = Tuple[str, str]+++async def append_memory_turns_atomic(+ database: Any,+ memory_node_id: str,+ turns: Iterable[MemoryTurn],+ *,+ window_size: int,+ mutation_id: Optional[str] = None,+ parameter_updates: Optional[Dict[str, Any]] = None,+ remove_parameters: Sequence[str] = (),+ replacement_content: Optional[str] = None,+ expected_content: Optional[str] = None,+) -> Tuple[Dict[str, Any], list[str], bool]:+ """Append one or more role/message pairs without stale overwrites.++ ``replacement_content`` is used for a compaction summary only when the+ stored content still equals ``expected_content``. If another writer has+ appended since compaction started, its newer content wins and this turn is+ appended to it instead of erasing it.++ The fallback path preserves compatibility with lightweight database mocks+ used by plugin tests; production :class:`core.database.Database` always+ provides ``mutate_node_parameters_atomic``.+ """+ materialized_turns = [(str(role), str(message)) for role, message in turns]+ updates = dict(parameter_updates or {})+ # Removed transcript blocks are needed only by the process that actually+ # applies this mutation (for optional long-term archival). Keep them out+ # of RuntimeMutation.result so trimming memory does not silently retain+ # the deleted conversation text in the idempotency ledger.+ removed_for_applied: list[str] = []++ def _transform(params: Dict[str, Any]):+ current_content = params.get("memory_content") or (+ "# Conversation History\n\n*No messages yet.*\n"+ )+ if (+ replacement_content is not None+ and (expected_content is None or current_content == expected_content)+ ):+ content = replacement_content+ else:+ content = current_content++ for role, message in materialized_turns:+ content = append_to_memory_markdown(content, role, message)+ content, removed_texts = trim_markdown_window(+ content,+ max(1, int(window_size)),+ )+ removed_for_applied[:] = removed_texts++ params.update(updates)+ for key in remove_parameters:+ params.pop(key, None)+ params["memory_content"] = content+ params.pop("memoryContent", None)+ return params, {+ "appended_turns": len(materialized_turns),+ "trimmed_count": len(removed_texts),+ }++ atomic_mutate = getattr(database, "mutate_node_parameters_atomic", None)+ if atomic_mutate is not None and inspect.iscoroutinefunction(atomic_mutate):+ params, _metadata, applied = await atomic_mutate(+ memory_node_id,+ _transform,+ mutation_id=mutation_id,+ operation="append_memory",+ )+ # The mutator closure is invoked only for a newly applied write. A+ # ledger retry skips it, naturally returning no blocks for duplicate+ # archival while the durable metadata remains content-free.+ removed_texts = list(removed_for_applied) if applied else []+ return params, removed_texts, applied++ # Compatibility for existing isolated unit-test doubles. This remains a+ # read/modify/write only when no production atomic API is available.+ params = await database.get_node_parameters(memory_node_id) or {}+ params, metadata = _transform(dict(params))+ await database.save_node_parameters(memory_node_id, params)+ return params, list(removed_for_applied), True+++async def update_memory_parameters_atomic(+ database: Any,+ memory_node_id: str,+ *,+ parameter_updates: Optional[Dict[str, Any]] = None,+ remove_parameters: Sequence[str] = (),+ mutation_id: Optional[str] = None,+) -> Tuple[Dict[str, Any], bool]:+ """Atomically update non-transcript fields on a memory node."""+ updates = dict(parameter_updates or {})++ def _transform(params: Dict[str, Any]):+ params.update(updates)+ for key in remove_parameters:+ params.pop(key, None)+ return params++ atomic_mutate = getattr(database, "mutate_node_parameters_atomic", None)+ if atomic_mutate is not None and inspect.iscoroutinefunction(atomic_mutate):+ params, _metadata, applied = await atomic_mutate(+ memory_node_id,+ _transform,+ mutation_id=mutation_id,+ operation="update_memory_state",+ )+ return params, applied++ params = await database.get_node_parameters(memory_node_id) or {}+ params = _transform(dict(params))+ await database.save_node_parameters(memory_node_id, params)+ return params, Truediff --git a/server/services/memory/state.py b/server/services/memory/state.pyindex d0933e7d..d9ffeb82 100644--- a/server/services/memory/state.py+++ b/server/services/memory/state.py@@ -10,7 +10,7 @@ accumulated todos every run).When a ``memory_node_id`` is provided, the simpleMemory node's ownfields (``memory_content``, ``memory_jsonl``, ``last_session_id``) are-also reset to defaults via ``database.save_node_parameters`` so the+also reset through the atomic parameter mutation path so theclaude_code_agent JSONL bridge surface clears alongside."""@@ -32,10 +32,9 @@ async def clear_agent_session_state() -> Dict[str, Any]:"""Clear every store keyed by an agent's conversational scope.- ``TodoService`` is keyed by ``ctx.workflow_id or ctx.node_id or- "default"`` (see ``server/nodes/tool/write_todos.py``). We clear all- three candidate keys to match whichever fallback the agent actually- used at write time.+ Modern ``TodoService`` entries are keyed by workflow + writeTodos node.+ Clearing a workflow removes every v2 node-scoped key plus the historical+ workflow/session/default fallbacks used by legacy callers.Args:session_id: ``simpleMemory`` node's ``session_id`` parameter.@@ -60,7 +59,7 @@ async def clear_agent_session_state(# consolidate). Lazy import keeps ``services.ai``'s heavy LangChain# deps off the hot path.from services.ai import _memory_vector_stores as _live_vector_stores- from services.todo_service import get_todo_service+ from services.todo_service import UNSAVED_WORKFLOW_ID, get_todo_servicecleared_vector_store = Falseif clear_long_term and session_id in _live_vector_stores:@@ -70,6 +69,11 @@ async def clear_agent_session_state(todo_service = get_todo_service()cleared_todo_keys: List[str] = []++ cleared_todo_keys.extend(+ todo_service.clear_workflow(workflow_id or UNSAVED_WORKFLOW_ID)+ )+seen = set()for key in (workflow_id, session_id, "default"):if key and key not in seen:@@ -82,20 +86,24 @@ async def clear_agent_session_state(from services.plugin.deps import get_databasedb = get_database()- params = await db.get_node_parameters(memory_node_id) or {}- params["memory_content"] = DEFAULT_MEMORY_CONTENT- # Wipe `last_session_id` so claude_code_agent starts a fresh- # session on next spawn instead of `--resume`-ing into the now-- # cleared transcript. Drop any orphan JSONL field from the- # earlier bridge revision.- params["last_session_id"] = None- params.pop("memory_jsonl", None)- # Wipe the Vertex managed-agent chain so the next run starts a- # fresh interaction + sandbox instead of continuing the cleared- # conversation (vertex_managed_agent memory bridge).- params.pop("vertex_interaction_id", None)- params.pop("vertex_environment_id", None)- await db.save_node_parameters(memory_node_id, params)+ from services.memory.runtime import update_memory_parameters_atomic++ # This shares the same BEGIN IMMEDIATE transaction path as runtime+ # appends, preventing a clear from restoring a stale parameter+ # snapshot over unrelated concurrent updates.+ await update_memory_parameters_atomic(+ db,+ memory_node_id,+ parameter_updates={+ "memory_content": DEFAULT_MEMORY_CONTENT,+ "last_session_id": None,+ },+ remove_parameters=(+ "memory_jsonl",+ "vertex_interaction_id",+ "vertex_environment_id",+ ),+ )cleared_memory_node = Truelogger.info("[Memory] Cleared simpleMemory node fields memory_node=%s " "(memory_content reset + last_session_id wiped)",diff --git a/server/services/proxy/service.py b/server/services/proxy/service.pyindex 4f7ee7d4..8c68de46 100644--- a/server/services/proxy/service.py+++ b/server/services/proxy/service.py@@ -5,6 +5,7 @@ scoring, geo-targeting, sticky sessions, failover, and cost tracking.Follows the CompactionService singleton pattern."""+import asyncioimport jsonimport timefrom collections import deque@@ -134,6 +135,11 @@ class ProxyService:self._settings = settingsself._providers: Dict[str, _ProviderRuntime] = {}self._routing_rules: List[RoutingRule] = []+ # Serialize fetch + async credential/build + pointer swap. Atomic+ # assignment prevents partial snapshots, while these locks prevent an+ # older slow reload from publishing after a newer one.+ self._provider_reload_lock = asyncio.Lock()+ self._routing_reload_lock = asyncio.Lock()self._daily_spend_usd: float = 0.0self._initialized = False@@ -145,11 +151,10 @@ class ProxyService:"""try:# Load provider configs from database- db_providers = await self._database.get_proxy_providers()- await self._load_providers(db_providers)+ await self.reload_providers()# Load routing rules from database- await self._reload_routing_rules()+ await self.reload_routing_rules()self._initialized = Trueprovider_names = [p.config.name for p in self._providers.values() if p.config.enabled]@@ -286,14 +291,14 @@ class ProxyService:"""Save a routing rule to database and refresh in-memory list."""success = await self._database.save_proxy_routing_rule(rule_data)if success:- await self._reload_routing_rules()+ await self.reload_routing_rules()return successasync def delete_routing_rule(self, rule_id: int) -> bool:"""Delete a routing rule and refresh in-memory list."""success = await self._database.delete_proxy_routing_rule(rule_id)if success:- await self._reload_routing_rules()+ await self.reload_routing_rules()return successasync def test_provider(self, provider_name: str) -> Dict[str, Any]:@@ -357,13 +362,25 @@ class ProxyService:async def reload_providers(self) -> None:"""Reload provider configs from database (called after tool/UI changes)."""- db_providers = await self._database.get_proxy_providers()- await self._load_providers(db_providers)+ async with self._provider_reload_lock:+ db_providers = await self._database.get_proxy_providers()+ await self._load_providers(db_providers)logger.info("Proxy providers reloaded", count=len(self._providers))+ async def reload_routing_rules(self) -> None:+ """Reload routing rules after an out-of-service database write."""+ async with self._routing_reload_lock:+ await self._reload_routing_rules()+ logger.info("Proxy routing rules reloaded", count=len(self._routing_rules))+async def _load_providers(self, db_providers: list) -> None:- """Build _ProviderRuntime instances from database rows."""- self._providers.clear()+ """Build and atomically publish a complete provider snapshot.++ Credential lookups await. Mutating ``self._providers`` in place used+ to expose an empty or half-built routing table to requests arriving+ during a reload.+ """+ providers: Dict[str, _ProviderRuntime] = {}for row in db_providers:template = json.loads(row.get("url_template", "{}"))provider = TemplateProxyProvider(name=row["name"], template=template)@@ -395,7 +412,12 @@ class ProxyService:else:logger.warning("No credentials for proxy provider", name=config.name)- self._providers[config.name] = runtime+ providers[config.name] = runtime++ # A dict reference assignment is atomic for readers in this event+ # loop; every request now observes either the old complete snapshot+ # or the new complete snapshot.+ self._providers = providers# ---- Internal helpers ----@@ -438,11 +460,11 @@ class ProxyService:return candidates[0]async def _reload_routing_rules(self) -> None:- """Reload routing rules from database."""+ """Reload and atomically publish a complete routing-rule snapshot."""db_rules = await self._database.get_proxy_routing_rules()- self._routing_rules = []+ routing_rules: List[RoutingRule] = []for row in db_rules:- self._routing_rules.append(+ routing_rules.append(RoutingRule(id=row.get("id"),domain_pattern=row["domain_pattern"],@@ -456,7 +478,8 @@ class ProxyService:priority=row.get("priority", 0),))- self._routing_rules.sort(key=lambda r: r.priority)+ routing_rules.sort(key=lambda r: r.priority)+ self._routing_rules = routing_rules# ---- Singleton accessor (CompactionService pattern) ----diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.pyindex f5ddf2eb..41713562 100644--- a/server/services/temporal/agent_activities.py+++ b/server/services/temporal/agent_activities.py@@ -243,23 +243,32 @@ async def persist_agent_turn(payload: Dict[str, Any]) -> Dict[str, Any]:return {"appended": False, "trimmed_count": 0}from core.container import container- from services.memory.markdown import (- append_to_memory_markdown,- trim_markdown_window,- )+ from services.memory.runtime import append_memory_turns_atomicdatabase = container.database()- params = await database.get_node_parameters(memory_node_id) or {}- current = params.get("memory_content", "")-- updated = append_to_memory_markdown(current, "human", payload.get("human_text", ""))- updated = append_to_memory_markdown(updated, "ai", payload.get("assistant_text", ""))-- window_size = int(payload.get("window_size", 10))- trimmed_content, trimmed_pairs = trim_markdown_window(updated, window_size)+ mutation_id = payload.get("mutation_id")+ if not mutation_id:+ # Temporal keeps activity_id stable across automatic retries, making+ # it a durable idempotency key without changing workflow payloads.+ try:+ info = activity.info()+ mutation_id = (+ f"temporal-memory:{info.workflow_id}:{info.activity_id}:"+ f"{memory_node_id}"+ )+ except Exception: # pragma: no cover - direct unit invocation+ mutation_id = None- params["memory_content"] = trimmed_content- await database.save_node_parameters(memory_node_id, params)+ params, trimmed_pairs, applied = await append_memory_turns_atomic(+ database,+ memory_node_id,+ [+ ("human", payload.get("human_text", "")),+ ("ai", payload.get("assistant_text", "")),+ ],+ window_size=int(payload.get("window_size", 10)),+ mutation_id=mutation_id,+ )# Broadcast so the parameter panel auto-refetches mid-run.# CloudEvents v1.0 envelope (RFC §6.4) — type is@@ -271,13 +280,18 @@ async def persist_agent_turn(payload: Dict[str, Any]) -> Dict[str, Any]:from services.status_broadcaster import get_status_broadcasterbroadcaster = get_status_broadcaster()- await broadcaster.broadcast_node_parameters_updated(- memory_node_id,- parameters=params,- source_hint="agent",- )+ if applied:+ await broadcaster.broadcast_node_parameters_updated(+ memory_node_id,+ parameters=params,+ source_hint="agent",+ )- return {"appended": True, "trimmed_count": len(trimmed_pairs)}+ return {+ "appended": applied,+ "applied": applied,+ "trimmed_count": len(trimmed_pairs),+ }@activity.defn(name="agent.compact_memory.v1")