Skip to content

Self-improving AI Employees for everyone

fix(runtime): make shared node state mutations atomic

trohitg16b15c42e438
diff --git a/server/core/database.py b/server/core/database.py
index 9082f4da..92685ecd 100644
--- a/server/core/database.py
+++ b/server/core/database.py
@@ -1,18 +1,21 @@
"""Modern async database service with SQLModel and SQLAlchemy 2.0."""
import json
+import inspect
+from copy import deepcopy
from datetime import datetime, timezone
-from typing import Dict, Any, List, Optional
+from typing import Awaitable, Callable, Dict, Any, List, Optional, Tuple, Union
from pydantic_core import to_jsonable_python
from sqlmodel import SQLModel, select
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
from sqlalchemy.exc import IntegrityError
-from sqlalchemy import text, func
+from sqlalchemy import case, text, func, update
from contextlib import asynccontextmanager
from core.config import Settings
from models.database import (
NodeParameter,
+ RuntimeMutation,
Workflow,
Execution,
APIKey,
@@ -40,6 +43,18 @@ from core.logging import get_logger
logger = get_logger(__name__)
+RuntimeMutationCallback = Callable[
+ [AsyncSession],
+ Union[Optional[Dict[str, Any]], Awaitable[Optional[Dict[str, Any]]]],
+]
+ParameterMutator = Callable[
+ [Dict[str, Any]],
+ Union[
+ Dict[str, Any],
+ Tuple[Dict[str, Any], Optional[Dict[str, Any]]],
+ ],
+]
+
class Database:
"""Async database service with SQLModel."""
@@ -231,6 +246,165 @@ class Database:
finally:
await session.close()
+ async def run_runtime_mutation(
+ self,
+ *,
+ resource_type: str,
+ resource_id: str,
+ operation: str,
+ mutate: RuntimeMutationCallback,
+ mutation_id: Optional[str] = None,
+ ) -> Tuple[Dict[str, Any], bool]:
+ """Run one shared-state mutation under a reserved write transaction.
+
+ SQLite's normal deferred transactions allow two workers to read the
+ same JSON value before either writes it. ``BEGIN IMMEDIATE`` obtains
+ the write reservation *before* the read, serialising the complete
+ read/modify/write cycle. When ``mutation_id`` is supplied, the
+ durable ledger row is committed in that same transaction and a retry
+ returns the original result with ``applied=False``.
+
+ The callback receives the transaction's :class:`AsyncSession` and
+ must not commit it. This method also works with non-SQLite test or
+ future production databases, using their regular transaction begin.
+ """
+ if not self.async_session:
+ raise RuntimeError("Database not initialized")
+
+ async with self.async_session() as session:
+ try:
+ if self.engine is not None and self.engine.dialect.name == "sqlite":
+ await session.execute(text("BEGIN IMMEDIATE"))
+ else:
+ await session.begin()
+
+ if mutation_id:
+ found = await session.execute(
+ select(RuntimeMutation).where(
+ RuntimeMutation.mutation_id == mutation_id,
+ RuntimeMutation.resource_type == resource_type,
+ RuntimeMutation.resource_id == resource_id,
+ )
+ )
+ existing = found.scalar_one_or_none()
+ if existing is not None:
+ result = deepcopy(existing.result or {})
+ await session.rollback()
+ return result, False
+
+ result_or_awaitable = mutate(session)
+ if inspect.isawaitable(result_or_awaitable):
+ result = await result_or_awaitable
+ else:
+ result = result_or_awaitable
+ result = deepcopy(result or {})
+
+ if mutation_id:
+ session.add(
+ RuntimeMutation(
+ mutation_id=mutation_id,
+ resource_type=resource_type,
+ resource_id=resource_id,
+ operation=operation,
+ result=result,
+ )
+ )
+
+ await session.commit()
+ return result, True
+ except Exception:
+ await session.rollback()
+ raise
+
+ async def mutate_node_parameters_atomic(
+ self,
+ node_id: str,
+ mutator: ParameterMutator,
+ *,
+ mutation_id: Optional[str] = None,
+ operation: str = "update",
+ ) -> Tuple[Dict[str, Any], Dict[str, Any], bool]:
+ """Atomically transform one node's parameters.
+
+ ``mutator`` receives an isolated copy and may return either the new
+ parameter dict or ``(new_parameters, result_metadata)``. The final
+ persisted parameters, metadata, and whether this call applied the
+ write are returned.
+ """
+
+ async def _mutate(session: AsyncSession) -> Dict[str, Any]:
+ selected = await session.execute(
+ select(NodeParameter).where(NodeParameter.node_id == node_id)
+ )
+ row = selected.scalar_one_or_none()
+ current = deepcopy(row.parameters if row is not None else {})
+ transformed = mutator(current)
+ if isinstance(transformed, tuple):
+ new_parameters, metadata = transformed
+ else:
+ new_parameters, metadata = transformed, {}
+ if not isinstance(new_parameters, dict):
+ raise TypeError("node parameter mutator must return a dict")
+ if row is None:
+ session.add(
+ NodeParameter(
+ node_id=node_id,
+ parameters=deepcopy(new_parameters),
+ )
+ )
+ else:
+ row.parameters = deepcopy(new_parameters)
+ row.updated_at = datetime.now(timezone.utc)
+ return deepcopy(metadata or {})
+
+ metadata, applied = await self.run_runtime_mutation(
+ resource_type="node_parameters",
+ resource_id=node_id,
+ operation=operation,
+ mutation_id=mutation_id,
+ mutate=_mutate,
+ )
+ parameters = await self.get_node_parameters(node_id) or {}
+ return parameters, metadata, applied
+
+ async def mutate_workflow_data_atomic(
+ self,
+ workflow_id: str,
+ mutator: ParameterMutator,
+ *,
+ mutation_id: Optional[str] = None,
+ operation: str = "update",
+ ) -> Tuple[Optional[Workflow], Dict[str, Any], bool]:
+ """Atomically transform ``Workflow.data`` without stale overwrites."""
+
+ async def _mutate(session: AsyncSession) -> Dict[str, Any]:
+ selected = await session.execute(
+ select(Workflow).where(Workflow.id == workflow_id)
+ )
+ workflow = selected.scalar_one_or_none()
+ if workflow is None:
+ return {"found": False}
+ transformed = mutator(deepcopy(workflow.data or {}))
+ if isinstance(transformed, tuple):
+ new_data, metadata = transformed
+ else:
+ new_data, metadata = transformed, {}
+ if not isinstance(new_data, dict):
+ raise TypeError("workflow data mutator must return a dict")
+ workflow.data = deepcopy(new_data)
+ workflow.updated_at = datetime.now(timezone.utc)
+ return {"found": True, **deepcopy(metadata or {})}
+
+ metadata, applied = await self.run_runtime_mutation(
+ resource_type="workflow",
+ resource_id=workflow_id,
+ operation=operation,
+ mutation_id=mutation_id,
+ mutate=_mutate,
+ )
+ workflow = await self.get_workflow(workflow_id)
+ return workflow, metadata, applied
+
# ============================================================================
# Node Parameters
# ============================================================================
@@ -2300,60 +2474,146 @@ class Database:
return []
async def claim_task(self, task_id: str, agent_node_id: str) -> Optional[Dict[str, Any]]:
- """Claim a pending task. Returns None if already claimed."""
+ """Atomically claim a pending task; same-agent retries are safe."""
try:
async with self.get_session() as session:
- result = await session.execute(
- select(TeamTask).where(TeamTask.id == task_id, TeamTask.status == "pending", TeamTask.assigned_to.is_(None))
+ claimed = await session.execute(
+ update(TeamTask)
+ .where(
+ TeamTask.id == task_id,
+ TeamTask.status == "pending",
+ TeamTask.assigned_to.is_(None),
+ )
+ .values(
+ assigned_to=agent_node_id,
+ status="in_progress",
+ started_at=datetime.now(timezone.utc),
+ )
)
- task = result.scalar_one_or_none()
- if not task:
+ # The guarded UPDATE is the compare-and-swap. It remains
+ # correct on databases where a SELECT + Python mutation would
+ # race because there is no SQLite BEGIN IMMEDIATE reservation.
+ selected = await session.execute(
+ select(TeamTask).where(TeamTask.id == task_id)
+ )
+ task = selected.scalar_one_or_none()
+ if task is None:
+ await session.rollback()
return None
- task.assigned_to = agent_node_id
- task.status = "in_progress"
- task.started_at = datetime.now(timezone.utc)
+
+ won = bool(claimed.rowcount == 1)
+ same_agent_retry = (
+ task.status == "in_progress"
+ and task.assigned_to == agent_node_id
+ )
+ if not won and not same_agent_retry:
+ await session.rollback()
+ return None
+
await session.commit()
- return {"id": task.id, "title": task.title, "assigned_to": agent_node_id}
+ return {
+ "id": task.id,
+ "title": task.title,
+ "assigned_to": agent_node_id,
+ }
except Exception as e:
logger.error(f"Failed to claim task: {e}")
return None
async def complete_task(self, task_id: str, result_data: Optional[Dict[str, Any]] = None) -> bool:
- """Mark task as completed."""
- try:
- async with self.get_session() as session:
- result = await session.execute(select(TeamTask).where(TeamTask.id == task_id))
- task = result.scalar_one_or_none()
- if not task:
- return False
- task.status = "completed"
- task.result = result_data
- task.progress = 100
- task.completed_at = datetime.now(timezone.utc)
- await session.commit()
- return True
+ """Atomically transition an in-progress task to completed."""
+ try:
+ async def _complete(session: AsyncSession) -> Dict[str, Any]:
+ completed = await session.execute(
+ update(TeamTask)
+ .where(
+ TeamTask.id == task_id,
+ TeamTask.status == "in_progress",
+ )
+ .values(
+ status="completed",
+ result=result_data,
+ progress=100,
+ completed_at=datetime.now(timezone.utc),
+ )
+ )
+ if completed.rowcount == 1:
+ return {"success": True}
+
+ selected = await session.execute(
+ select(TeamTask).where(TeamTask.id == task_id)
+ )
+ task = selected.scalar_one_or_none()
+ return {
+ "success": bool(
+ task is not None and task.status == "completed"
+ )
+ }
+
+ result, _applied = await self.run_runtime_mutation(
+ resource_type="team_task",
+ resource_id=task_id,
+ operation="complete",
+ mutate=_complete,
+ )
+ return bool(result.get("success"))
except Exception as e:
logger.error(f"Failed to complete task: {e}")
return False
async def fail_task(self, task_id: str, error: str) -> bool:
- """Mark task as failed."""
- try:
- async with self.get_session() as session:
- result = await session.execute(select(TeamTask).where(TeamTask.id == task_id))
- task = result.scalar_one_or_none()
- if not task:
- return False
- task.error = error
- task.retry_count += 1
- if task.retry_count < task.max_retries:
- task.status = "pending"
- task.assigned_to = None
- else:
- task.status = "failed"
- task.completed_at = datetime.now(timezone.utc)
- await session.commit()
- return True
+ """Atomically record one failed attempt and release or fail it."""
+ try:
+ async def _fail(session: AsyncSession) -> Dict[str, Any]:
+ next_retry = TeamTask.retry_count + 1
+ retryable = next_retry < TeamTask.max_retries
+ failed = await session.execute(
+ update(TeamTask)
+ .where(
+ TeamTask.id == task_id,
+ TeamTask.status == "in_progress",
+ )
+ .values(
+ error=error,
+ retry_count=next_retry,
+ status=case(
+ (retryable, "pending"),
+ else_="failed",
+ ),
+ assigned_to=case(
+ (retryable, None),
+ else_=TeamTask.assigned_to,
+ ),
+ completed_at=case(
+ (retryable, TeamTask.completed_at),
+ else_=datetime.now(timezone.utc),
+ ),
+ )
+ )
+ if failed.rowcount == 1:
+ return {"success": True}
+
+ selected = await session.execute(
+ select(TeamTask).where(TeamTask.id == task_id)
+ )
+ task = selected.scalar_one_or_none()
+ # A transport retry after this exact failure observes the
+ # released/terminal state and must not increment twice.
+ return {
+ "success": bool(
+ task is not None
+ and task.status in {"pending", "failed"}
+ and task.error == error
+ )
+ }
+
+ result, _applied = await self.run_runtime_mutation(
+ resource_type="team_task",
+ resource_id=task_id,
+ operation="fail",
+ mutate=_fail,
+ )
+ return bool(result.get("success"))
except Exception as e:
logger.error(f"Failed to fail task: {e}")
return False
diff --git a/server/models/database.py b/server/models/database.py
index 25801053..ee0f9736 100644
--- a/server/models/database.py
+++ b/server/models/database.py
@@ -22,6 +22,31 @@ class NodeParameter(SQLModel, table=True):
)
+class RuntimeMutation(SQLModel, table=True):
+ """Durable idempotency ledger for shared runtime state mutations.
+
+ Runtime workers (including Temporal activity retries) can execute the
+ same logical write more than once. Keeping the mutation key in the
+ same SQLite transaction as the state change makes those retries a
+ no-op, even across processes or server restarts.
+ """
+
+ __tablename__ = "runtime_mutations"
+
+ # The idempotency identity is scoped to the mutated resource. A caller
+ # may legitimately reuse a transport-level mutation id for two different
+ # nodes; those writes must not suppress one another.
+ mutation_id: str = Field(primary_key=True, max_length=512)
+ resource_type: str = Field(primary_key=True, max_length=100)
+ resource_id: str = Field(primary_key=True, max_length=512)
+ operation: str = Field(default="update", max_length=100)
+ result: Optional[Dict[str, Any]] = Field(default=None, sa_column=Column(JSON))
+ created_at: datetime = Field(
+ default_factory=lambda: datetime.now(timezone.utc),
+ sa_column=Column(DateTime(timezone=True), server_default=func.now()),
+ )
+
+
class Workflow(SQLModel, table=True):
"""Workflow definitions.
diff --git a/server/nodes/agent/claude_code_agent/__init__.py b/server/nodes/agent/claude_code_agent/__init__.py
index c55309f8..d17eec14 100644
--- a/server/nodes/agent/claude_code_agent/__init__.py
+++ b/server/nodes/agent/claude_code_agent/__init__.py
@@ -389,6 +389,7 @@ class ClaudeCodeAgentNode(ActionNode):
connected_skill_names=connected_skills,
connected_tools=tool_data,
connected_memory=memory_data,
+ execution_id=ctx.execution_id,
allowed_credentials=params.allowed_credentials,
max_parallel=params.max_parallel,
)
diff --git a/server/nodes/agent/claude_code_agent/_pool.py b/server/nodes/agent/claude_code_agent/_pool.py
index 793290bc..2a8d83d6 100644
--- a/server/nodes/agent/claude_code_agent/_pool.py
+++ b/server/nodes/agent/claude_code_agent/_pool.py
@@ -147,6 +147,11 @@ class PooledClaudeSession:
# watcher picks up add/remove events without respawning.
materialised_skills: frozenset = field(default_factory=frozenset)
last_used_at: float = field(default_factory=time.monotonic)
+ # Lease held from ``acquire`` through context rebinding, ``send_turn``,
+ # and ``release``. The narrower ``lock`` below only protected stdin;
+ # without this lease a second batch could replace the MCP BatchContext
+ # after the first batch acquired the session but before its turn ran.
+ turn_lease: asyncio.Lock = field(default_factory=asyncio.Lock)
# Per-session lock; held throughout a turn. The reaper consults
# ``lock.locked()`` to skip in-flight sessions.
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
@@ -246,141 +251,190 @@ class ClaudeSessionPool:
Caller MUST call :meth:`release` so the reaper can mark idle.
"""
- async with self._pool_lock:
- existing = self._pool.get(memory_node_id)
- crashed_uuid = ""
- if existing is not None and existing.process.returncode is not None:
- logger.info(
- "[ClaudeSessionPool] dropping dead session " "memory_node=%s pid=%s exit=%s — will respawn",
- memory_node_id,
- existing.process.pid,
- existing.process.returncode,
- )
- crashed_uuid = existing.current_session_uuid
- await self._terminate_locked(memory_node_id, reason="crashed")
- existing = None
-
- if existing is not None:
- existing.last_used_at = time.monotonic()
- # Rebind the persistent :class:`BatchContext` bound to
- # the warm subprocess's argv-baked bearer token to the
- # surface the current batch declares. Without this the
- # per-handler scope check in
- # ``workflow_tools._build_handler`` still sees the
- # stale ``connected_tools`` from the spawn batch and
- # lets disconnected tools fire (and FastMCP refcounts
- # leak by 1 per batch indefinitely).
- #
- # Service.py just issued ``mcp_bearer_token`` (T_new)
- # for this batch and called ``register_batch`` on it.
- # We transplant T_new's BatchContext values onto
- # T_warm (the argv-baked one), then drop T_new — it
- # would otherwise sit orphaned in ``_active_tokens``
- # until app shutdown.
- if existing.batch_token and mcp_bearer_token and mcp_bearer_token != existing.batch_token:
- from services.cli_agent.mcp_server import (
- lookup_batch,
- rebind_batch,
- unregister_batch,
+ while True:
+ # Capture or create the entry under the global map lock, but do
+ # not wait for a busy per-memory lease here: a queued turn for M1
+ # must never prevent an unrelated M2 from being acquired.
+ async with self._pool_lock:
+ existing = self._pool.get(memory_node_id)
+ crashed_uuid = ""
+ if existing is not None and existing.process.returncode is not None:
+ logger.info(
+ "[ClaudeSessionPool] dropping dead session "
+ "memory_node=%s pid=%s exit=%s — will respawn",
+ memory_node_id,
+ existing.process.pid,
+ existing.process.returncode,
)
-
- new_ctx = lookup_batch(mcp_bearer_token)
- if new_ctx is not None:
- rebind_batch(
- existing.batch_token,
- connected_tools=new_ctx.connected_tools,
- connected_skill_names=new_ctx.connected_skill_names,
- allowed_credentials=new_ctx.allowed_credentials,
- workspace_dir=new_ctx.workspace_dir,
- )
- unregister_batch(mcp_bearer_token)
- # Live skill-tree update — claude live-watches
- # ``<workspace_dir>/.claude/skills/`` so add/remove
- # takes effect within the current session per the
- # skills spec's "Live change detection" rule
- # (code.claude.com/docs/en/skills#live-change-detection).
- # Same diff semantics as the MCP rebind above:
- # ``apply_skill_diff`` walks (added, removed) against
- # ``existing.materialised_skills`` and only touches
- # the delta on disk.
- if existing.workspace_dir is not None:
- from ._skills import materialise_skills
-
- new_set = frozenset(connected_skill_names or [])
- added, removed = await materialise_skills(
- existing.workspace_dir,
- new_set,
- previous_skill_names=existing.materialised_skills,
- log_label=f"pool {memory_node_id}",
+ crashed_uuid = existing.current_session_uuid
+ await self._terminate_locked(memory_node_id, reason="crashed")
+ existing = None
+
+ if existing is None:
+ if len(self._pool) >= self._max_size:
+ await self._evict_lru_locked()
+
+ # When recovering from a crash, splice the captured UUID
+ # into the spec so the spawn emits ``--resume <UUID>``.
+ if crashed_uuid and not spec.resume_session_id:
+ spec.resume_session_id = crashed_uuid
+ spec.continue_session = False
+
+ session = await self._spawn(
+ memory_node_id=memory_node_id,
+ spec=spec,
+ cwd=cwd,
+ env=env,
+ defaults=defaults,
+ mcp_endpoint_url=mcp_endpoint_url,
+ mcp_bearer_token=mcp_bearer_token,
+ connected_tool_names=connected_tool_names,
+ connected_skill_names=connected_skill_names,
+ workspace_dir=workspace_dir,
)
- if added or removed:
+ if mcp_bearer_token:
+ session.batch_token = mcp_bearer_token
+ session.workspace_dir = workspace_dir
+ session.materialised_skills = frozenset(
+ connected_skill_names or []
+ )
+ if crashed_uuid:
+ session.current_session_uuid = crashed_uuid
+ self._pool[memory_node_id] = session
+ await session.turn_lease.acquire() # new; never contended
+ try:
logger.info(
- "[ClaudeSessionPool] skill diff " "memory_node=%s +%d -%d (now %d wired)",
+ "[ClaudeSessionPool] spawned new session memory_node=%s pid=%s",
memory_node_id,
- added,
- removed,
- len(new_set),
+ session.process.pid,
)
- existing.materialised_skills = new_set
- logger.info(
- "[ClaudeSessionPool] warm reuse memory_node=%s pid=%s uuid=%s",
- memory_node_id,
- existing.process.pid,
- existing.current_session_uuid or "(unresolved)",
- )
- return existing
-
- if len(self._pool) >= self._max_size:
- await self._evict_lru_locked()
-
- # When recovering from a crash, splice the captured UUID
- # into the spec so the spawn emits ``--resume <UUID>``.
- if crashed_uuid and not spec.resume_session_id:
- spec.resume_session_id = crashed_uuid
- spec.continue_session = False
+ await self._emit_event(
+ "spawned",
+ memory_node_id=memory_node_id,
+ workflow_id=workflow_id,
+ session_uuid=session.current_session_uuid,
+ pid=session.process.pid,
+ )
+ except BaseException:
+ if session.turn_lease.locked():
+ session.turn_lease.release()
+ await self._terminate_locked(
+ memory_node_id,
+ reason="acquire_failed",
+ )
+ raise
+ return session
- session = await self._spawn(
+ # ``existing`` was captured while holding the map lock. Wait only
+ # on its own lease, then verify it was not evicted/replaced while
+ # we waited before rebinding any context.
+ await existing.turn_lease.acquire()
+ try:
+ async with self._pool_lock:
+ still_current = (
+ self._pool.get(memory_node_id) is existing
+ and existing.process.returncode is None
+ )
+ except BaseException:
+ # Cancellation can land while waiting for the map lock after
+ # this turn already acquired its per-memory lease.
+ if existing.turn_lease.locked():
+ existing.turn_lease.release()
+ raise
+ if not still_current:
+ existing.turn_lease.release()
+ continue
+
+ return await self._prepare_warm_reuse(
+ existing,
memory_node_id=memory_node_id,
- spec=spec,
- cwd=cwd,
- env=env,
- defaults=defaults,
- mcp_endpoint_url=mcp_endpoint_url,
mcp_bearer_token=mcp_bearer_token,
- connected_tool_names=connected_tool_names,
connected_skill_names=connected_skill_names,
- workspace_dir=workspace_dir,
)
- # Track the spawn-time bearer token so subsequent batches can
- # rebind the persistent BatchContext in place (warm-reuse
- # path above) and ``_terminate_locked`` can drain FastMCP
- # tool refcounts on death.
- if mcp_bearer_token:
- session.batch_token = mcp_bearer_token
- # Track the workspace + initial skill set so warm-reuse
- # paths know where to materialise live updates and what
- # the prior set was for the diff.
- session.workspace_dir = workspace_dir
- session.materialised_skills = frozenset(connected_skill_names or [])
- if crashed_uuid:
- # The new subprocess is resuming the prior session; pre-seed
- # the UUID so the first turn's emitted CloudEvents carry
- # the right id even before the result event lands.
- session.current_session_uuid = crashed_uuid
- self._pool[memory_node_id] = session
+
+ async def _prepare_warm_reuse(
+ self,
+ existing: PooledClaudeSession,
+ *,
+ memory_node_id: str,
+ mcp_bearer_token: Optional[str],
+ connected_skill_names: Optional[List[str]],
+ ) -> PooledClaudeSession:
+ """Rebind one already-leased warm process for the next turn."""
+ try:
+ existing.last_used_at = time.monotonic()
+ if (
+ existing.batch_token
+ and mcp_bearer_token
+ and mcp_bearer_token != existing.batch_token
+ ):
+ from services.cli_agent.mcp_server import (
+ lookup_batch,
+ rebind_batch,
+ unregister_batch,
+ )
+
+ new_ctx = lookup_batch(mcp_bearer_token)
+ if new_ctx is not None:
+ rebind_batch(
+ existing.batch_token,
+ workflow_id=new_ctx.workflow_id,
+ node_id=new_ctx.node_id,
+ execution_id=new_ctx.execution_id,
+ broadcaster=new_ctx.broadcaster,
+ connected_tools=new_ctx.connected_tools,
+ connected_skill_names=new_ctx.connected_skill_names,
+ allowed_credentials=new_ctx.allowed_credentials,
+ workspace_dir=new_ctx.workspace_dir,
+ )
+ unregister_batch(mcp_bearer_token)
+
+ if existing.workspace_dir is not None:
+ from ._skills import materialise_skills
+
+ new_set = frozenset(connected_skill_names or [])
+ added, removed = await materialise_skills(
+ existing.workspace_dir,
+ new_set,
+ previous_skill_names=existing.materialised_skills,
+ log_label=f"pool {memory_node_id}",
+ )
+ if added or removed:
+ logger.info(
+ "[ClaudeSessionPool] skill diff "
+ "memory_node=%s +%d -%d (now %d wired)",
+ memory_node_id,
+ added,
+ removed,
+ len(new_set),
+ )
+ existing.materialised_skills = new_set
+
logger.info(
- "[ClaudeSessionPool] spawned new session memory_node=%s pid=%s",
+ "[ClaudeSessionPool] warm reuse memory_node=%s pid=%s uuid=%s",
memory_node_id,
- session.process.pid,
- )
- await self._emit_event(
- "spawned",
- memory_node_id=memory_node_id,
- workflow_id=workflow_id,
- session_uuid=session.current_session_uuid,
- pid=session.process.pid,
+ existing.process.pid,
+ existing.current_session_uuid or "(unresolved)",
)
- return session
+ return existing
+ except BaseException:
+ # ``acquire`` did not return a session, so its caller has no
+ # object it can pass to ``release``. Also discard the fresh token
+ # allocated for a failed warm rebind; service.py deliberately
+ # leaves pool-path token ownership to this class.
+ if (
+ mcp_bearer_token
+ and mcp_bearer_token != existing.batch_token
+ ):
+ try:
+ from services.cli_agent.mcp_server import unregister_batch
+
+ unregister_batch(mcp_bearer_token)
+ except Exception:
+ pass
+ if existing.turn_lease.locked():
+ existing.turn_lease.release()
+ raise
async def clear(
self,
@@ -413,8 +467,10 @@ class ClaudeSessionPool:
return ""
async def release(self, session: PooledClaudeSession) -> None:
- """Update ``last_used_at`` so the reaper measures from now."""
+ """Release the context+turn lease and start the idle clock."""
session.last_used_at = time.monotonic()
+ if session.turn_lease.locked():
+ session.turn_lease.release()
async def terminate(self, memory_node_id: str) -> None:
"""Force-drop a specific pooled session."""
@@ -889,7 +945,11 @@ class ClaudeSessionPool:
async def _evict_lru_locked(self) -> None:
"""Caller holds ``self._pool_lock``. Evicts the least-recently-
used non-in-flight entry."""
- idle_entries = [(key, sess) for key, sess in self._pool.items() if not sess.lock.locked()]
+ idle_entries = [
+ (key, sess)
+ for key, sess in self._pool.items()
+ if not sess.lock.locked() and not sess.turn_lease.locked()
+ ]
if not idle_entries:
logger.info(
"[ClaudeSessionPool] all %d entries in-flight; pool over cap",
@@ -992,7 +1052,11 @@ class ClaudeSessionPool:
now = time.monotonic()
async with self._pool_lock:
expired_keys = [
- key for key, sess in self._pool.items() if not sess.lock.locked() and now - sess.last_used_at >= self._idle_ttl
+ key
+ for key, sess in self._pool.items()
+ if not sess.lock.locked()
+ and not sess.turn_lease.locked()
+ and now - sess.last_used_at >= self._idle_ttl
]
for key in expired_keys:
logger.info(
diff --git a/server/nodes/agent/vertex_managed_agent/__init__.py b/server/nodes/agent/vertex_managed_agent/__init__.py
index 0c9a4fe5..67959519 100644
--- a/server/nodes/agent/vertex_managed_agent/__init__.py
+++ b/server/nodes/agent/vertex_managed_agent/__init__.py
@@ -328,14 +328,25 @@ class VertexManagedAgentNode(ActionNode):
prev_interaction_id
and cause is not None
and (is_expired_environment_error(cause) or is_precondition_failure(cause))
- ):
+ ):
logger.warning(
"[Vertex Agent] unresumable interaction chain for %s (%s) — retrying fresh",
node_id,
str(cause).replace("\n", " ")[:120],
)
if memory_node_id:
- await self._save_chain_ids(database, memory_node_id, None, None)
+ await self._save_chain_ids(
+ database,
+ memory_node_id,
+ None,
+ None,
+ mutation_id=(
+ f"vertex-chain-reset:{ctx.execution_id}:"
+ f"{node_id}:{memory_node_id}"
+ if ctx.execution_id
+ else None
+ ),
+ )
prev_interaction_id = None
environment = "remote"
interaction = await run_turn(input=prompt, environment="remote")
@@ -441,6 +452,11 @@ class VertexManagedAgentNode(ActionNode):
human=prompt,
assistant=response_text,
window_size=int((memory_data or {}).get("window_size") or 10),
+ mutation_id=(
+ f"vertex-memory:{ctx.execution_id}:{node_id}:{memory_node_id}"
+ if ctx.execution_id
+ else None
+ ),
)
# ---- usage bookkeeping (tokens billed by Google; recorded for stats)
@@ -986,28 +1002,40 @@ class VertexManagedAgentNode(ActionNode):
human: Optional[str] = None,
assistant: Optional[str] = None,
window_size: int = 10,
+ mutation_id: Optional[str] = None,
) -> None:
"""Persist chain ids (and optionally the turn) onto simpleMemory."""
- from services.memory.markdown import (
- append_to_memory_markdown,
- trim_markdown_window,
+ from services.memory.runtime import (
+ append_memory_turns_atomic,
+ update_memory_parameters_atomic,
)
- params = await database.get_node_parameters(memory_node_id) or {}
+ updates: Dict[str, Any] = {}
+ removals: List[str] = []
if interaction_id:
- params["vertex_interaction_id"] = interaction_id
+ updates["vertex_interaction_id"] = interaction_id
else:
- params.pop("vertex_interaction_id", None)
+ removals.append("vertex_interaction_id")
if environment_id:
- params["vertex_environment_id"] = environment_id
+ updates["vertex_environment_id"] = environment_id
else:
- params.pop("vertex_environment_id", None)
+ removals.append("vertex_environment_id")
if human and assistant:
- content = params.get("memory_content") or ""
- content = append_to_memory_markdown(content, "human", human)
- content = append_to_memory_markdown(content, "assistant", assistant)
- content, _removed = trim_markdown_window(content, window_size)
- params["memory_content"] = content
-
- await database.save_node_parameters(memory_node_id, params)
+ await append_memory_turns_atomic(
+ database,
+ memory_node_id,
+ [("human", human), ("assistant", assistant)],
+ window_size=window_size,
+ mutation_id=mutation_id,
+ parameter_updates=updates,
+ remove_parameters=removals,
+ )
+ else:
+ await update_memory_parameters_atomic(
+ database,
+ memory_node_id,
+ parameter_updates=updates,
+ remove_parameters=removals,
+ mutation_id=mutation_id,
+ )
diff --git a/server/nodes/proxy/proxy_config/__init__.py b/server/nodes/proxy/proxy_config/__init__.py
index f62892ba..609d75c2 100644
--- a/server/nodes/proxy/proxy_config/__init__.py
+++ b/server/nodes/proxy/proxy_config/__init__.py
@@ -320,7 +320,7 @@ async def _add_routing_rule(p: Dict[str, Any], proxy_svc) -> Dict[str, Any]:
}
)
if proxy_svc:
- await proxy_svc.reload_providers()
+ await proxy_svc.reload_routing_rules()
return {"success": True, "domain_pattern": domain_pattern}
@@ -333,7 +333,7 @@ async def _remove_routing_rule(p: Dict[str, Any], proxy_svc) -> Dict[str, Any]:
db = get_database()
await db.delete_proxy_routing_rule(int(rule_id))
if proxy_svc:
- await proxy_svc.reload_providers()
+ await proxy_svc.reload_routing_rules()
return {"success": True, "rule_id": rule_id}
diff --git a/server/nodes/tool/agent_builder/__init__.py b/server/nodes/tool/agent_builder/__init__.py
index 8cab5c38..cf6ca267 100644
--- a/server/nodes/tool/agent_builder/__init__.py
+++ b/server/nodes/tool/agent_builder/__init__.py
@@ -30,6 +30,9 @@ to call something that isn't there yet.
from __future__ import annotations
+import hashlib
+import inspect
+import json
import uuid
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional
@@ -76,7 +79,118 @@ _CREATE_WORKFLOW_ENABLED = False
# ----------------------------------------------------------------------------
-async def _persist_canvas_mutation(workflow_id: Optional[str], ops: List[Dict[str, Any]]) -> None:
+def _apply_canvas_ops(
+ data: Dict[str, Any],
+ ops: List[Dict[str, Any]],
+) -> tuple[Dict[str, Any], Dict[str, Any]]:
+ """Apply an agentBuilder batch to one current workflow snapshot.
+
+ Minted node IDs and exact edge identities are de-duplicated so retrying
+ the same batch is harmless. Distinct batches retain their distinct minted
+ nodes even when they happen to add the same type concurrently.
+ """
+ data = dict(data or {})
+ nodes = [dict(node) for node in (data.get("nodes") or [])]
+ edges = [dict(edge) for edge in (data.get("edges") or [])]
+ ref_to_id: Dict[str, str] = {}
+ changed = False
+
+ def _resolve(ref: Any) -> Optional[str]:
+ if isinstance(ref, str):
+ return ref
+ if isinstance(ref, dict) and "client_ref" in ref:
+ return ref_to_id.get(ref["client_ref"])
+ return None
+
+ for op in ops:
+ op_type = op.get("type")
+ if op_type == "add_node":
+ node_type = op.get("node_type") or ""
+ if not node_type:
+ continue
+ new_id = op.get("minted_id") or op.get("client_ref") or ""
+ if not new_id:
+ continue
+ client_ref = op.get("client_ref") or ""
+
+ if client_ref:
+ ref_to_id[client_ref] = new_id
+ if any(node.get("id") == new_id for node in nodes):
+ continue
+ params = op.get("parameters") or {}
+ nodes.append(
+ {
+ "id": new_id,
+ "type": node_type,
+ "position": (
+ op.get("position")
+ if isinstance(op.get("position"), dict)
+ and "x" in op["position"]
+ else {"x": 0, "y": 0}
+ ),
+ "data": {
+ "label": op.get("label") or params.get("label") or node_type,
+ "parameters": dict(params),
+ },
+ }
+ )
+ changed = True
+ elif op_type == "add_edge":
+ source = _resolve(op.get("source"))
+ target = _resolve(op.get("target"))
+ if not source or not target:
+ continue
+ identity = (
+ source,
+ target,
+ op.get("source_handle"),
+ op.get("target_handle"),
+ )
+ if any(
+ (
+ edge.get("source"),
+ edge.get("target"),
+ edge.get("sourceHandle"),
+ edge.get("targetHandle"),
+ )
+ == identity
+ for edge in edges
+ ):
+ continue
+ edges.append(
+ {
+ "id": f"e-{source}-{target}",
+ "source": source,
+ "target": target,
+ "sourceHandle": op.get("source_handle"),
+ "targetHandle": op.get("target_handle"),
+ }
+ )
+ changed = True
+ elif op_type == "set_node_parameters":
+ node_id = op.get("node_id")
+ merge = op.get("parameters") or {}
+ for node in nodes:
+ if node.get("id") != node_id:
+ continue
+ node_data = dict(node.get("data") or {})
+ existing = dict(node_data.get("parameters") or {})
+ merged = {**existing, **merge}
+ if merged != existing:
+ node_data["parameters"] = merged
+ node["data"] = node_data
+ changed = True
+ break
+
+ data["nodes"] = nodes
+ data["edges"] = edges
+ return data, {"changed": changed, "nodes": len(nodes), "edges": len(edges)}
+
+
+async def _persist_canvas_mutation(
+ workflow_id: Optional[str],
+ ops: List[Dict[str, Any]],
+) -> Optional[bool]:
"""Apply the three op types agentBuilder emits to the persisted
``workflow.data`` via the existing ``database.get_workflow`` +
``database.save_workflow`` round-trip.
@@ -91,85 +205,60 @@ async def _persist_canvas_mutation(workflow_id: Optional[str], ops: List[Dict[st
ops for FE alignment) so the persisted row matches the React Flow
state exactly — no id divergence on reload.
"""
- if not ops or not workflow_id:
- return
+ if not ops:
+ return False
+ if not workflow_id:
+ return None
try:
from services.plugin.deps import get_database
database = get_database()
+ atomic_mutate = getattr(database, "mutate_workflow_data_atomic", None)
+ if atomic_mutate is not None and inspect.iscoroutinefunction(atomic_mutate):
+ canonical_ops = json.dumps(
+ ops,
+ sort_keys=True,
+ separators=(",", ":"),
+ default=str,
+ )
+ mutation_id = (
+ f"agent-builder:{workflow_id}:"
+ f"{hashlib.sha256(canonical_ops.encode('utf-8')).hexdigest()}"
+ )
+ workflow, metadata, applied = await atomic_mutate(
+ workflow_id,
+ lambda current: _apply_canvas_ops(current, ops),
+ mutation_id=mutation_id,
+ operation="agent_builder_ops",
+ )
+ if workflow is None or not metadata.get("found", True):
+ logger.debug(
+ "[agentBuilder] persist skipped: workflow %s not found",
+ workflow_id,
+ )
+ return None
+ logger.info(
+ "[agentBuilder] transactionally persisted %s ops to workflow %s "
+ "(nodes=%d edges=%d applied=%s)",
+ [op.get("type") for op in ops],
+ workflow_id,
+ metadata.get("nodes", 0),
+ metadata.get("edges", 0),
+ applied,
+ )
+ return bool(applied and metadata.get("changed"))
+
+ # Compatibility path for existing isolated test doubles.
workflow = await database.get_workflow(workflow_id)
if workflow is None:
logger.debug("[agentBuilder] persist skipped: workflow %s not found", workflow_id)
- return
-
- data = dict(workflow.data or {})
- nodes = list(data.get("nodes") or [])
- edges = list(data.get("edges") or [])
- ref_to_id: Dict[str, str] = {}
-
- def _resolve(ref: Any) -> Optional[str]:
- if isinstance(ref, str):
- return ref
- if isinstance(ref, dict) and "client_ref" in ref:
- return ref_to_id.get(ref["client_ref"])
return None
- for op in ops:
- op_type = op.get("type")
- if op_type == "add_node":
- node_type = op.get("node_type") or ""
- if not node_type:
- continue
- # ``minted_id`` is set by add_tool / add_skill / add_subagent
- # so the FE applier adopts the same id. Reuse it for the
- # persisted row to keep BE/FE views aligned.
- new_id = op.get("minted_id") or op.get("client_ref") or ""
- if not new_id:
- continue
- client_ref = op.get("client_ref") or ""
- if client_ref:
- ref_to_id[client_ref] = new_id
- params = op.get("parameters") or {}
- nodes.append(
- {
- "id": new_id,
- "type": node_type,
- "position": op.get("position") if isinstance(op.get("position"), dict) and "x" in op["position"] else {"x": 0, "y": 0},
- "data": {
- "label": op.get("label") or params.get("label") or node_type,
- "parameters": dict(params),
- },
- }
- )
- elif op_type == "add_edge":
- source = _resolve(op.get("source"))
- target = _resolve(op.get("target"))
- if not source or not target:
- continue
- edges.append(
- {
- "id": f"e-{source}-{target}",
- "source": source,
- "target": target,
- "sourceHandle": op.get("source_handle"),
- "targetHandle": op.get("target_handle"),
- }
- )
- elif op_type == "set_node_parameters":
- node_id = op.get("node_id")
- merge = op.get("parameters") or {}
- for n in nodes:
- if n.get("id") != node_id:
- continue
- node_data = dict(n.get("data") or {})
- existing = dict(node_data.get("parameters") or {})
- existing.update(merge)
- node_data["parameters"] = existing
- n["data"] = node_data
- break
-
- data["nodes"] = nodes
- data["edges"] = edges
+ data, metadata = _apply_canvas_ops(dict(workflow.data or {}), ops)
+ nodes = data.get("nodes") or []
+ edges = data.get("edges") or []
+ if not metadata.get("changed"):
+ return False
await database.save_workflow(
workflow_id=workflow_id,
name=workflow.name,
@@ -184,10 +273,12 @@ async def _persist_canvas_mutation(workflow_id: Optional[str], ops: List[Dict[st
len(nodes),
len(edges),
)
+ return True
except Exception as exc:
logger.warning(
"[agentBuilder] persist failed for workflow %s: %s", workflow_id, exc, exc_info=True
)
+ return None
async def _broadcast(workflow_id: Optional[str], caller_id: str, ops: List[Dict[str, Any]]) -> None:
@@ -203,7 +294,12 @@ async def _broadcast(workflow_id: Optional[str], caller_id: str, ops: List[Dict[
"""
if not ops:
return
- await _persist_canvas_mutation(workflow_id, ops)
+ persisted = await _persist_canvas_mutation(workflow_id, ops)
+ # ``None`` means persistence was unavailable/missing and is explicitly
+ # best-effort: the live canvas still needs the operation event. ``False``
+ # is a known idempotent no-op and should not be rebroadcast.
+ if persisted is False:
+ return
try:
from ._events import broadcast_workflow_ops
diff --git a/server/services/agent_team.py b/server/services/agent_team.py
index 7b7c3cfa..b13a882c 100644
--- a/server/services/agent_team.py
+++ b/server/services/agent_team.py
@@ -130,11 +130,14 @@ class AgentTeamService:
async def claim_task(self, team_id: str, task_id: str, agent_node_id: str) -> Optional[Dict[str, Any]]:
"""Claim a task for an agent."""
- # Update member status to working
- await self.database.update_member_status(team_id, agent_node_id, "working")
-
task = await self.database.claim_task(task_id, agent_node_id)
+ # Only the winner of the conditional database claim becomes working.
+ # Updating first left every losing parallel contender stuck in the
+ # working state even though exactly one owned the task.
+ if task:
+ await self.database.update_member_status(team_id, agent_node_id, "working")
+
if task and self.broadcaster:
await self.broadcaster.broadcast_team_event(team_id, "task_claimed", {**task, "claimed_by": agent_node_id})
diff --git a/server/services/ai.py b/server/services/ai.py
index 4490db2d..264211c9 100644
--- a/server/services/ai.py
+++ b/server/services/ai.py
@@ -41,6 +41,7 @@ import json
# those typed errors cleanly through ``BaseNode.execute()``.
import openai
from services.plugin import NodeUserError
+from services.tool_identity import DuplicateToolNameError, ensure_unique_tool_names
if TYPE_CHECKING:
from langchain_openai import ChatOpenAI # noqa: F401
@@ -851,6 +852,13 @@ async def _run_agent_loop(
added = await rebind_from_operations(result["operations"])
if added:
iteration_new_tools.extend(added)
+ except DuplicateToolNameError as exc:
+ # Reject the whole hot-rebind batch before mutating the
+ # bound surface. Return the structured conflict to the
+ # model so it can rename/rebuild the ambiguous tool.
+ logger.warning("[Agent loop] rejected ambiguous tool rebind: %s", exc)
+ result = dict(result)
+ result["rebind_error"] = exc.as_dict()
except Exception as exc: # noqa: BLE001 — defensive; rebind is opt-in
logger.warning(
"[Agent loop] rebind_from_operations raised: %s", exc, exc_info=True
@@ -1731,6 +1739,8 @@ class AIService:
# Build tools if provided
tools = []
tool_configs = {}
+ tool_identities: List[Dict[str, str]] = []
+ tool_bindings: List[tuple[Any, Dict[str, Any]]] = []
if tool_data:
await broadcast_status("building_tools", {"message": f"Building {len(tool_data)} tool(s)...", "tool_count": len(tool_data)})
@@ -1739,13 +1749,20 @@ class AIService:
tool, config = await self._build_tool_from_node(tool_info)
if tool:
tools.append(tool)
- tool_configs[tool.name] = config
+ tool_bindings.append((tool, config))
+ tool_identities.append(
+ {
+ "name": tool.name,
+ "node_id": str(config.get("node_id") or tool_info.get("node_id") or ""),
+ "label": str(config.get("label") or tool_info.get("label") or tool_info.get("node_type") or "tool"),
+ }
+ )
logger.debug(f"[Agent] Registered tool: name={tool.name}, node_id={config.get('node_id')}")
logger.debug(f"[Agent] Built {len(tools)} tools")
# Auto-inject check_delegated_tasks tool when delegation tools present
- if any(name.startswith("delegate_to_") for name in tool_configs):
+ if any(tool.name.startswith("delegate_to_") for tool in tools):
check_info = {
"node_type": "_builtin_check_delegated_tasks",
"node_id": f"{node_id}_check_tasks",
@@ -1755,11 +1772,18 @@ class AIService:
check_tool, check_config = await self._build_tool_from_node(check_info)
if check_tool:
tools.append(check_tool)
- tool_configs[check_tool.name] = check_config
+ tool_bindings.append((check_tool, check_config))
+ tool_identities.append(
+ {
+ "name": check_tool.name,
+ "node_id": str(check_config.get("node_id") or check_info["node_id"]),
+ "label": str(check_config.get("label") or check_info["label"]),
+ }
+ )
logger.debug("[Agent] Auto-injected check_delegated_tasks tool")
# Add delegation guidance to system message
- delegate_names = [n for n in tool_configs if n.startswith("delegate_to_")]
+ delegate_names = [tool.name for tool in tools if tool.name.startswith("delegate_to_")]
system_message += (
"\n\n## Agent Delegation\n"
"When delegating to sub-agents, use 'task' for the mission directive "
@@ -1767,6 +1791,12 @@ class AIService:
f"Available agents: {', '.join(delegate_names)}"
)
+ # Name-based dispatch is only safe after validating the full
+ # surface. Build the lookup map after this check so a later
+ # duplicate can never silently replace an earlier node.
+ ensure_unique_tool_names(tool_identities)
+ tool_configs.update({tool.name: config for tool, config in tool_bindings})
+
# Now that ``system_message`` is final (skill prompt + delegation
# guidance both folded in), prepend it as the first message.
# See the comment at the original ``initial_messages`` declaration
@@ -1860,7 +1890,7 @@ class AIService:
"""
from services.node_registry import get_node_class
- new_tools: List[Any] = []
+ new_bindings: List[tuple[Any, Dict[str, Any]]] = []
for op in operations:
if op.get("type") != "add_node":
continue
@@ -1896,10 +1926,21 @@ class AIService:
continue
if tool is None:
continue
- new_tools.append(tool)
- if tool_config:
- tool_configs[tool.name] = tool_config
- return new_tools
+ new_bindings.append((tool, tool_config or tool_info))
+
+ new_identities = [
+ {
+ "name": tool.name,
+ "node_id": str(config.get("node_id") or ""),
+ "label": str(config.get("label") or config.get("node_type") or "tool"),
+ }
+ for tool, config in new_bindings
+ ]
+ ensure_unique_tool_names([*tool_identities, *new_identities])
+ for tool, tool_config in new_bindings:
+ tool_configs[tool.name] = tool_config
+ tool_identities.extend(new_identities)
+ return [tool for tool, _ in new_bindings]
# Broadcast: Building agent
await broadcast_status(
@@ -2014,22 +2055,33 @@ class AIService:
},
)
- # If compaction happened, use compacted summary as base
+ from services.memory.runtime import append_memory_turns_atomic
+
+ # Compaction may take long enough for another worker to append.
+ # The atomic helper installs the summary only if the content it
+ # compacted is still current; otherwise it preserves the newer
+ # transcript and appends this exchange to it.
+ compacted_summary = None
if compaction_result and compaction_result.get("success") and compaction_result.get("summary"):
- # Start fresh with compacted summary, then add current exchange
- updated_content = compaction_result["summary"]
- updated_content = _append_to_memory_markdown(updated_content, "human", prompt)
- updated_content = _append_to_memory_markdown(updated_content, "ai", response_content)
+ compacted_summary = compaction_result["summary"]
logger.info("[Agent Memory] Using compacted summary as new base")
- else:
- # Normal flow: append to existing memory
- updated_content = memory_data.get("memory_content", "# Conversation History\n\n*No messages yet.*\n")
- updated_content = _append_to_memory_markdown(updated_content, "human", prompt)
- updated_content = _append_to_memory_markdown(updated_content, "ai", response_content)
- # Trim to window size, archive removed to vector DB
- window_size = memory_data.get("window_size", 10)
- updated_content, removed_texts = _trim_markdown_window(updated_content, window_size)
+ memory_node_id = memory_data["node_id"]
+ execution_id = (context or {}).get("execution_id")
+ mutation_id = (
+ f"ai-memory:{execution_id}:{node_id}:{memory_node_id}"
+ if execution_id
+ else None
+ )
+ _current_params, removed_texts, _applied = await append_memory_turns_atomic(
+ self.database,
+ memory_node_id,
+ [("human", prompt), ("ai", response_content)],
+ window_size=int(memory_data.get("window_size", 10)),
+ mutation_id=mutation_id,
+ replacement_content=compacted_summary,
+ expected_content=memory_data.get("memory_content", ""),
+ )
# Store removed messages in long-term vector DB
if removed_texts and memory_data.get("long_term_enabled"):
@@ -2041,14 +2093,6 @@ class AIService:
except Exception as e:
logger.warning(f"[Agent Memory] Failed to archive to vector store: {e}")
- # Save updated markdown to node parameters. Schema-
- # canonical key is snake_case; drop any pre-migration
- # camelCase mirror so the saved params stay clean.
- memory_node_id = memory_data["node_id"]
- current_params = await self.database.get_node_parameters(memory_node_id) or {}
- current_params["memory_content"] = updated_content
- current_params.pop("memoryContent", None)
- await self.database.save_node_parameters(memory_node_id, current_params)
logger.info(f"[Agent Memory] Saved markdown to memory node '{memory_node_id}'")
result = {
@@ -2082,6 +2126,17 @@ class AIService:
"execution_time": time.time() - start_time,
}
+ except DuplicateToolNameError as e:
+ logger.warning("[Agent] Duplicate tool surface rejected: %s", e)
+ return {
+ "success": False,
+ "node_id": node_id,
+ "node_type": "aiAgent",
+ **e.as_dict(),
+ "execution_time": time.time() - start_time,
+ "timestamp": datetime.now().isoformat(),
+ }
+
except openai.OpenAIError as e:
# See execute_chat for the rationale — typed SDK errors are
# user-correctable and re-raised as NodeUserError so BaseNode
@@ -2167,6 +2222,8 @@ class AIService:
# This supports ALL tool types: calculatorTool, currentTimeTool, duckduckgoSearch, androidTool, httpRequest
all_tools = []
tool_node_configs = {} # Map tool name to node config (same as AI Agent's tool_configs)
+ tool_identities: List[Dict[str, str]] = []
+ tool_bindings: List[tuple[Any, Dict[str, Any]]] = []
if tool_data:
await broadcast_status("building_tools", {"message": f"Building {len(tool_data)} tool(s)...", "tool_count": len(tool_data)})
@@ -2175,7 +2232,14 @@ class AIService:
tool, config = await self._build_tool_from_node(tool_info)
if tool:
all_tools.append(tool)
- tool_node_configs[tool.name] = config
+ tool_bindings.append((tool, config))
+ tool_identities.append(
+ {
+ "name": tool.name,
+ "node_id": str(config.get("node_id") or tool_info.get("node_id") or ""),
+ "label": str(config.get("label") or tool_info.get("label") or tool_info.get("node_type") or "tool"),
+ }
+ )
logger.debug(
f"[ChatAgent] Built tool: {tool.name} (type={config.get('node_type')}, node_id={config.get('node_id')})"
)
@@ -2183,7 +2247,7 @@ class AIService:
logger.debug(f"[ChatAgent] Built {len(all_tools)} tools from tool_data")
# Auto-inject check_delegated_tasks tool when delegation tools present
- if any(name.startswith("delegate_to_") for name in tool_node_configs):
+ if any(tool.name.startswith("delegate_to_") for tool in all_tools):
check_info = {
"node_type": "_builtin_check_delegated_tasks",
"node_id": f"{node_id}_check_tasks",
@@ -2193,11 +2257,18 @@ class AIService:
check_tool, check_config = await self._build_tool_from_node(check_info)
if check_tool:
all_tools.append(check_tool)
- tool_node_configs[check_tool.name] = check_config
+ tool_bindings.append((check_tool, check_config))
+ tool_identities.append(
+ {
+ "name": check_tool.name,
+ "node_id": str(check_config.get("node_id") or check_info["node_id"]),
+ "label": str(check_config.get("label") or check_info["label"]),
+ }
+ )
logger.debug("[ChatAgent] Auto-injected check_delegated_tasks tool")
# Add delegation guidance to system message
- delegate_names = [n for n in tool_node_configs if n.startswith("delegate_to_")]
+ delegate_names = [tool.name for tool in all_tools if tool.name.startswith("delegate_to_")]
system_message += (
"\n\n## Agent Delegation\n"
"When delegating to sub-agents, use 'task' for the mission directive "
@@ -2205,6 +2276,9 @@ class AIService:
f"Available agents: {', '.join(delegate_names)}"
)
+ ensure_unique_tool_names(tool_identities)
+ tool_node_configs.update({tool.name: config for tool, config in tool_bindings})
+
logger.debug(f"[ChatAgent] Total tools available: {len(all_tools)}")
# Debug: log all tool schemas to verify they're correct
for t in all_tools:
@@ -2389,7 +2463,7 @@ class AIService:
(``tool_configs`` vs ``tool_node_configs``)."""
from services.node_registry import get_node_class
- new_tools: List[Any] = []
+ new_bindings: List[tuple[Any, Dict[str, Any]]] = []
for op in operations:
if op.get("type") != "add_node":
continue
@@ -2424,10 +2498,21 @@ class AIService:
continue
if tool is None:
continue
- new_tools.append(tool)
- if tool_config:
- tool_node_configs[tool.name] = tool_config
- return new_tools
+ new_bindings.append((tool, tool_config or tool_info))
+
+ new_identities = [
+ {
+ "name": tool.name,
+ "node_id": str(config.get("node_id") or ""),
+ "label": str(config.get("label") or config.get("node_type") or "tool"),
+ }
+ for tool, config in new_bindings
+ ]
+ ensure_unique_tool_names([*tool_identities, *new_identities])
+ for tool, tool_config in new_bindings:
+ tool_node_configs[tool.name] = tool_config
+ tool_identities.extend(new_identities)
+ return [tool for tool, _ in new_bindings]
# Run the agent loop. See ``execute_agent`` for the
# rationale on ``recursion_limit`` + the truncation
@@ -2513,21 +2598,29 @@ class AIService:
"saving_memory", {"message": "Saving to conversation memory...", "session_id": session_id, "has_memory": True}
)
- # If compaction happened, use compacted summary as base
+ from services.memory.runtime import append_memory_turns_atomic
+
+ compacted_summary = None
if compaction_result and compaction_result.get("success") and compaction_result.get("summary"):
- updated_content = compaction_result["summary"]
- updated_content = _append_to_memory_markdown(updated_content, "human", prompt)
- updated_content = _append_to_memory_markdown(updated_content, "ai", response_content)
+ compacted_summary = compaction_result["summary"]
logger.info("[ChatAgent Memory] Using compacted summary as new base")
- else:
- # Normal flow: append to existing memory
- updated_content = memory_content or "# Conversation History\n\n*No messages yet.*\n"
- updated_content = _append_to_memory_markdown(updated_content, "human", prompt)
- updated_content = _append_to_memory_markdown(updated_content, "ai", response_content)
- # Trim to window size, archive removed to vector DB
- window_size = memory_data.get("window_size", 10)
- updated_content, removed_texts = _trim_markdown_window(updated_content, window_size)
+ memory_node_id = memory_data["node_id"]
+ execution_id = (context or {}).get("execution_id")
+ mutation_id = (
+ f"chat-memory:{execution_id}:{node_id}:{memory_node_id}"
+ if execution_id
+ else None
+ )
+ _current_params, removed_texts, _applied = await append_memory_turns_atomic(
+ self.database,
+ memory_node_id,
+ [("human", prompt), ("ai", response_content)],
+ window_size=int(memory_data.get("window_size", 10)),
+ mutation_id=mutation_id,
+ replacement_content=compacted_summary,
+ expected_content=memory_content,
+ )
# Store removed messages in long-term vector DB
if removed_texts and memory_data.get("long_term_enabled"):
@@ -2539,14 +2632,6 @@ class AIService:
except Exception as e:
logger.warning(f"[ChatAgent Memory] Failed to archive to vector store: {e}")
- # Save updated markdown to node parameters. Schema-
- # canonical key is snake_case; drop any pre-migration
- # camelCase mirror so the saved params stay clean.
- memory_node_id = memory_data["node_id"]
- current_params = await self.database.get_node_parameters(memory_node_id) or {}
- current_params["memory_content"] = updated_content
- current_params.pop("memoryContent", None)
- await self.database.save_node_parameters(memory_node_id, current_params)
logger.info(f"[ChatAgent Memory] Saved markdown to memory node '{memory_node_id}'")
# Determine agent type based on configuration
@@ -2597,6 +2682,17 @@ class AIService:
"execution_time": time.time() - start_time,
}
+ except DuplicateToolNameError as e:
+ logger.warning("[ChatAgent] Duplicate tool surface rejected: %s", e)
+ return {
+ "success": False,
+ "node_id": node_id,
+ "node_type": "chatAgent",
+ **e.as_dict(),
+ "execution_time": time.time() - start_time,
+ "timestamp": datetime.now().isoformat(),
+ }
+
except openai.OpenAIError as e:
# Typed SDK error — see execute_chat for rationale.
log_api_call(logger, provider, model, "chat_agent", False, error=str(e))
diff --git a/server/services/cli_agent/mcp_server.py b/server/services/cli_agent/mcp_server.py
index 48e5d6de..b83e423f 100644
--- a/server/services/cli_agent/mcp_server.py
+++ b/server/services/cli_agent/mcp_server.py
@@ -36,6 +36,7 @@ from starlette.requests import Request
from starlette.responses import JSONResponse, Response
from core.logging import get_logger
+from services.tool_identity import ensure_unique_tool_names
logger = get_logger(__name__)
@@ -57,6 +58,7 @@ class BatchContext:
workflow_id: str
node_id: str
workspace_dir: Path
+ execution_id: Optional[str] = None
connected_skill_names: Set[str] = field(default_factory=set)
allowed_credentials: Set[str] = field(default_factory=set)
# Connected ``input-tools`` nodes (entries from
@@ -73,6 +75,7 @@ class BatchContext:
# Token -> BatchContext registry. Lives in-memory only; tokens never
# touch disk or the credentials.db.
_active_tokens: Dict[str, BatchContext] = {}
+_UNSET = object()
def issue_token() -> str:
@@ -80,6 +83,19 @@ def issue_token() -> str:
return secrets.token_hex(32)
+def _validate_connected_tool_names(connected_tools: List[Dict[str, Any]]) -> None:
+ """Reject ambiguous node-type dispatch within one CLI agent surface."""
+ ensure_unique_tool_names(
+ {
+ "name": tool["node_type"],
+ "node_id": tool.get("node_id") or "",
+ "label": tool.get("label") or tool["node_type"],
+ }
+ for tool in connected_tools
+ if tool.get("node_type")
+ )
+
+
def register_batch(token: str, ctx: BatchContext) -> None:
"""Register a batch's auth token + expose its connected workflow
tools on the FastMCP server. Idempotent on identical context."""
@@ -88,6 +104,11 @@ def register_batch(token: str, ctx: BatchContext) -> None:
if existing is not ctx:
raise ValueError("Token collision in MCP server batch registry")
return
+ # Validate before either the bearer-token registry or FastMCP's global
+ # tool registry changes. The MCP handler dispatches by node_type, so two
+ # physical nodes of the same effective name would otherwise select the
+ # first one silently.
+ _validate_connected_tool_names(ctx.connected_tools)
_active_tokens[token] = ctx
tool_names = [t.get("node_type") for t in ctx.connected_tools]
logger.info(
@@ -125,6 +146,10 @@ def lookup_batch(token: str) -> Optional[BatchContext]:
def rebind_batch(
token: str,
*,
+ workflow_id: Optional[str] = None,
+ node_id: Optional[str] = None,
+ execution_id: Any = _UNSET,
+ broadcaster: Any = _UNSET,
connected_tools: Optional[List[Dict[str, Any]]] = None,
connected_skill_names: Optional[Set[str]] = None,
allowed_credentials: Optional[Set[str]] = None,
@@ -154,6 +179,19 @@ def rebind_batch(
ctx = _active_tokens.get(token)
if ctx is None:
return None
+ # Rebinding is transactional from the caller's perspective: an invalid
+ # tool surface must not partially refresh identity, credentials, or MCP
+ # refcounts on the warm process's persistent context.
+ if connected_tools is not None:
+ _validate_connected_tool_names(connected_tools)
+ if workflow_id is not None:
+ ctx.workflow_id = workflow_id
+ if node_id is not None:
+ ctx.node_id = node_id
+ if execution_id is not _UNSET:
+ ctx.execution_id = execution_id
+ if broadcaster is not _UNSET:
+ ctx.broadcaster = broadcaster
if connected_tools is not None:
from services.cli_agent.workflow_tools import (
expose_workflow_tools,
diff --git a/server/services/cli_agent/service.py b/server/services/cli_agent/service.py
index 55c42c2b..7c311a7e 100644
--- a/server/services/cli_agent/service.py
+++ b/server/services/cli_agent/service.py
@@ -78,6 +78,7 @@ class AICliService:
connected_skill_names: Optional[List[str]] = None,
connected_tools: Optional[List[Dict[str, Any]]] = None,
connected_memory: Optional[Dict[str, Any]] = None,
+ execution_id: Optional[str] = None,
allowed_credentials: Optional[List[str]] = None,
max_parallel: int = DEFAULT_MAX_PARALLEL,
mcp_port: Optional[int] = None,
@@ -172,6 +173,7 @@ class AICliService:
workflow_id=workflow_id,
node_id=node_id,
workspace_dir=Path(workspace_dir).resolve(),
+ execution_id=execution_id,
connected_skill_names=set(connected_skill_names or []),
allowed_credentials=set(allowed_credentials or []),
connected_tools=list(connected_tools or []),
@@ -318,6 +320,12 @@ class AICliService:
connected_memory,
results,
broadcaster=broadcaster,
+ mutation_id=(
+ f"cli-memory:{execution_id}:{node_id}:"
+ f"{connected_memory.get('node_id', '')}"
+ if execution_id
+ else None
+ ),
)
except Exception as exc: # pragma: no cover — best-effort
logger.warning(
@@ -461,19 +469,28 @@ class AICliService:
env["OPENCOMPANY_PARENT_RUN_ID"] = parent_run_id
env["MACHINA_PARENT_RUN_ID"] = parent_run_id # legacy child-process contract
- pooled = await pool.acquire(
- memory_node_id,
- spec=task,
- cwd=cwd,
- env=env,
- defaults=defaults,
- mcp_endpoint_url=mcp_endpoint_url,
- mcp_bearer_token=mcp_bearer_token,
- connected_tool_names=tool_names,
- connected_skill_names=connected_skill_names,
- workspace_dir=workspace_dir,
- workflow_id=workflow_id,
- )
+ try:
+ pooled = await pool.acquire(
+ memory_node_id,
+ spec=task,
+ cwd=cwd,
+ env=env,
+ defaults=defaults,
+ mcp_endpoint_url=mcp_endpoint_url,
+ mcp_bearer_token=mcp_bearer_token,
+ connected_tool_names=tool_names,
+ connected_skill_names=connected_skill_names,
+ workspace_dir=workspace_dir,
+ workflow_id=workflow_id,
+ )
+ except BaseException:
+ # A cold spawn can fail before the pool adopts the token. The
+ # run_batch pool branch intentionally does not unregister in its
+ # outer finally, so close this otherwise-orphaned context here.
+ from services.cli_agent.mcp_server import unregister_batch
+
+ unregister_batch(mcp_bearer_token)
+ raise
try:
return await pool.send_turn(
pooled,
@@ -504,8 +521,13 @@ class AICliService:
prior = params.get("last_session_id")
if not prior:
return # already cleared
- params["last_session_id"] = None
- await db.save_node_parameters(memory_node_id, params)
+ from services.memory.runtime import update_memory_parameters_atomic
+
+ await update_memory_parameters_atomic(
+ db,
+ memory_node_id,
+ parameter_updates={"last_session_id": None},
+ )
logger.warning(
"[CC-Agent _persist_memory] cleared stale last_session_id=%s "
"from memory_node=%s; next run will spawn a fresh claude "
@@ -519,6 +541,7 @@ class AICliService:
connected_memory: Dict[str, Any],
results: List[SessionResult],
broadcaster: Any = None,
+ mutation_id: Optional[str] = None,
) -> None:
"""Append each successful run's user prompt + assistant response
to ``simpleMemory.memory_content`` (markdown). Mirrors aiAgent /
@@ -554,38 +577,37 @@ class AICliService:
await AICliService._clear_stale_session_id(connected_memory)
return
- from services.memory import (
- append_to_memory_markdown,
- trim_markdown_window,
- )
+ from services.memory.runtime import append_memory_turns_atomic
from services.plugin.deps import get_database
db = get_database()
memory_node_id = connected_memory["node_id"]
- params = await db.get_node_parameters(memory_node_id) or {}
# 1. Persist claude's returned session_id from the most recent
# successful run. Drives `--resume <UUID>` on the next spawn so
# claude finds and continues its own JSONL transcript on disk.
last_run = next((r for r in reversed(successful) if r.session_id), None)
- if last_run is not None:
- params["last_session_id"] = last_run.session_id
-
- # 2. Update the user-visible markdown mirror.
- content = params.get("memory_content") or ("# Conversation History\n\n*No messages yet.*\n")
- for r in successful:
- content = append_to_memory_markdown(content, "human", r.prompt)
- content = append_to_memory_markdown(
- content,
- "ai",
- r.response or "",
- )
-
window = int(connected_memory.get("window_size") or 100)
- content, removed_texts = trim_markdown_window(content, window)
- params["memory_content"] = content
-
- await db.save_node_parameters(memory_node_id, params)
+ params, removed_texts, _applied = await append_memory_turns_atomic(
+ db,
+ memory_node_id,
+ [
+ turn
+ for result in successful
+ for turn in (
+ ("human", result.prompt),
+ ("ai", result.response or ""),
+ )
+ ],
+ window_size=window,
+ mutation_id=mutation_id,
+ parameter_updates=(
+ {"last_session_id": last_run.session_id}
+ if last_run is not None
+ else None
+ ),
+ )
+ content = params.get("memory_content") or ""
logger.info(
"[CC-Agent _persist_memory] saved memory_node=%s "
"last_session_id=%s appended_turns=%d archived_blocks=%d "
diff --git a/server/services/cli_agent/workflow_tools.py b/server/services/cli_agent/workflow_tools.py
index 3f6e7e3b..49977ee9 100644
--- a/server/services/cli_agent/workflow_tools.py
+++ b/server/services/cli_agent/workflow_tools.py
@@ -210,6 +210,9 @@ def _build_handler(node_type: str, params_cls: type):
"node_type": node_type,
"node_id": entry.get("node_id"),
"workflow_id": ctx.workflow_id,
+ "execution_id": ctx.execution_id,
+ "workspace_dir": str(ctx.workspace_dir),
+ "parent_node_id": ctx.node_id,
"label": entry.get("label") or node_type,
"parameters": dict(entry.get("parameters") or {}),
}
diff --git a/server/services/memory/__init__.py b/server/services/memory/__init__.py
index 56808193..69759ad9 100644
--- a/server/services/memory/__init__.py
+++ b/server/services/memory/__init__.py
@@ -32,6 +32,10 @@ from services.memory.jsonl import (
parse_jsonl,
trim_window,
)
+from services.memory.runtime import (
+ append_memory_turns_atomic,
+ update_memory_parameters_atomic,
+)
__all__ = [
# Markdown helpers — used by every agent bridge (aiAgent /
@@ -49,4 +53,7 @@ __all__ = [
"parse_jsonl",
"append_message",
"trim_window",
+ # Transactional runtime writes.
+ "append_memory_turns_atomic",
+ "update_memory_parameters_atomic",
]
diff --git a/server/services/memory/runtime.py b/server/services/memory/runtime.py
new file mode 100644
index 00000000..fd1d414a
--- /dev/null
+++ b/server/services/memory/runtime.py
@@ -0,0 +1,136 @@
+"""Atomic persistence helpers for runtime conversation memory.
+
+All agent bridges share the same ``simpleMemory`` JSON row. Appending by
+reading the row and later replacing it loses turns whenever two agents finish
+at the same time. This module keeps the markdown transformation inside the
+database's reserved write transaction and optionally records a durable
+idempotency key for workflow/activity retries.
+"""
+
+from __future__ import annotations
+
+import inspect
+from typing import Any, Dict, Iterable, Optional, Sequence, Tuple
+
+from services.memory.markdown import (
+ append_to_memory_markdown,
+ trim_markdown_window,
+)
+
+MemoryTurn = Tuple[str, str]
+
+
+async def append_memory_turns_atomic(
+ database: Any,
+ memory_node_id: str,
+ turns: Iterable[MemoryTurn],
+ *,
+ window_size: int,
+ mutation_id: Optional[str] = None,
+ parameter_updates: Optional[Dict[str, Any]] = None,
+ remove_parameters: Sequence[str] = (),
+ replacement_content: Optional[str] = None,
+ expected_content: Optional[str] = None,
+) -> Tuple[Dict[str, Any], list[str], bool]:
+ """Append one or more role/message pairs without stale overwrites.
+
+ ``replacement_content`` is used for a compaction summary only when the
+ stored content still equals ``expected_content``. If another writer has
+ appended since compaction started, its newer content wins and this turn is
+ appended to it instead of erasing it.
+
+ The fallback path preserves compatibility with lightweight database mocks
+ used by plugin tests; production :class:`core.database.Database` always
+ provides ``mutate_node_parameters_atomic``.
+ """
+ materialized_turns = [(str(role), str(message)) for role, message in turns]
+ updates = dict(parameter_updates or {})
+ # Removed transcript blocks are needed only by the process that actually
+ # applies this mutation (for optional long-term archival). Keep them out
+ # of RuntimeMutation.result so trimming memory does not silently retain
+ # the deleted conversation text in the idempotency ledger.
+ removed_for_applied: list[str] = []
+
+ def _transform(params: Dict[str, Any]):
+ current_content = params.get("memory_content") or (
+ "# Conversation History\n\n*No messages yet.*\n"
+ )
+ if (
+ replacement_content is not None
+ and (expected_content is None or current_content == expected_content)
+ ):
+ content = replacement_content
+ else:
+ content = current_content
+
+ for role, message in materialized_turns:
+ content = append_to_memory_markdown(content, role, message)
+ content, removed_texts = trim_markdown_window(
+ content,
+ max(1, int(window_size)),
+ )
+ removed_for_applied[:] = removed_texts
+
+ params.update(updates)
+ for key in remove_parameters:
+ params.pop(key, None)
+ params["memory_content"] = content
+ params.pop("memoryContent", None)
+ return params, {
+ "appended_turns": len(materialized_turns),
+ "trimmed_count": len(removed_texts),
+ }
+
+ atomic_mutate = getattr(database, "mutate_node_parameters_atomic", None)
+ if atomic_mutate is not None and inspect.iscoroutinefunction(atomic_mutate):
+ params, _metadata, applied = await atomic_mutate(
+ memory_node_id,
+ _transform,
+ mutation_id=mutation_id,
+ operation="append_memory",
+ )
+ # The mutator closure is invoked only for a newly applied write. A
+ # ledger retry skips it, naturally returning no blocks for duplicate
+ # archival while the durable metadata remains content-free.
+ removed_texts = list(removed_for_applied) if applied else []
+ return params, removed_texts, applied
+
+ # Compatibility for existing isolated unit-test doubles. This remains a
+ # read/modify/write only when no production atomic API is available.
+ params = await database.get_node_parameters(memory_node_id) or {}
+ params, metadata = _transform(dict(params))
+ await database.save_node_parameters(memory_node_id, params)
+ return params, list(removed_for_applied), True
+
+
+async def update_memory_parameters_atomic(
+ database: Any,
+ memory_node_id: str,
+ *,
+ parameter_updates: Optional[Dict[str, Any]] = None,
+ remove_parameters: Sequence[str] = (),
+ mutation_id: Optional[str] = None,
+) -> Tuple[Dict[str, Any], bool]:
+ """Atomically update non-transcript fields on a memory node."""
+ updates = dict(parameter_updates or {})
+
+ def _transform(params: Dict[str, Any]):
+ params.update(updates)
+ for key in remove_parameters:
+ params.pop(key, None)
+ return params
+
+ atomic_mutate = getattr(database, "mutate_node_parameters_atomic", None)
+ if atomic_mutate is not None and inspect.iscoroutinefunction(atomic_mutate):
+ params, _metadata, applied = await atomic_mutate(
+ memory_node_id,
+ _transform,
+ mutation_id=mutation_id,
+ operation="update_memory_state",
+ )
+ return params, applied
+
+ params = await database.get_node_parameters(memory_node_id) or {}
+ params = _transform(dict(params))
+ await database.save_node_parameters(memory_node_id, params)
+ return params, True
diff --git a/server/services/memory/state.py b/server/services/memory/state.py
index d0933e7d..d9ffeb82 100644
--- a/server/services/memory/state.py
+++ b/server/services/memory/state.py
@@ -10,7 +10,7 @@ accumulated todos every run).
When a ``memory_node_id`` is provided, the simpleMemory node's own
fields (``memory_content``, ``memory_jsonl``, ``last_session_id``) are
-also reset to defaults via ``database.save_node_parameters`` so the
+also reset through the atomic parameter mutation path so the
claude_code_agent JSONL bridge surface clears alongside.
"""
@@ -32,10 +32,9 @@ async def clear_agent_session_state(
) -> Dict[str, Any]:
"""Clear every store keyed by an agent's conversational scope.
- ``TodoService`` is keyed by ``ctx.workflow_id or ctx.node_id or
- "default"`` (see ``server/nodes/tool/write_todos.py``). We clear all
- three candidate keys to match whichever fallback the agent actually
- used at write time.
+ Modern ``TodoService`` entries are keyed by workflow + writeTodos node.
+ Clearing a workflow removes every v2 node-scoped key plus the historical
+ workflow/session/default fallbacks used by legacy callers.
Args:
session_id: ``simpleMemory`` node's ``session_id`` parameter.
@@ -60,7 +59,7 @@ async def clear_agent_session_state(
# consolidate). Lazy import keeps ``services.ai``'s heavy LangChain
# deps off the hot path.
from services.ai import _memory_vector_stores as _live_vector_stores
- from services.todo_service import get_todo_service
+ from services.todo_service import UNSAVED_WORKFLOW_ID, get_todo_service
cleared_vector_store = False
if clear_long_term and session_id in _live_vector_stores:
@@ -70,6 +69,11 @@ async def clear_agent_session_state(
todo_service = get_todo_service()
cleared_todo_keys: List[str] = []
+
+ cleared_todo_keys.extend(
+ todo_service.clear_workflow(workflow_id or UNSAVED_WORKFLOW_ID)
+ )
+
seen = set()
for key in (workflow_id, session_id, "default"):
if key and key not in seen:
@@ -82,20 +86,24 @@ async def clear_agent_session_state(
from services.plugin.deps import get_database
db = get_database()
- params = await db.get_node_parameters(memory_node_id) or {}
- params["memory_content"] = DEFAULT_MEMORY_CONTENT
- # Wipe `last_session_id` so claude_code_agent starts a fresh
- # session on next spawn instead of `--resume`-ing into the now-
- # cleared transcript. Drop any orphan JSONL field from the
- # earlier bridge revision.
- params["last_session_id"] = None
- params.pop("memory_jsonl", None)
- # Wipe the Vertex managed-agent chain so the next run starts a
- # fresh interaction + sandbox instead of continuing the cleared
- # conversation (vertex_managed_agent memory bridge).
- params.pop("vertex_interaction_id", None)
- params.pop("vertex_environment_id", None)
- await db.save_node_parameters(memory_node_id, params)
+ from services.memory.runtime import update_memory_parameters_atomic
+
+ # This shares the same BEGIN IMMEDIATE transaction path as runtime
+ # appends, preventing a clear from restoring a stale parameter
+ # snapshot over unrelated concurrent updates.
+ await update_memory_parameters_atomic(
+ db,
+ memory_node_id,
+ parameter_updates={
+ "memory_content": DEFAULT_MEMORY_CONTENT,
+ "last_session_id": None,
+ },
+ remove_parameters=(
+ "memory_jsonl",
+ "vertex_interaction_id",
+ "vertex_environment_id",
+ ),
+ )
cleared_memory_node = True
logger.info(
"[Memory] Cleared simpleMemory node fields memory_node=%s " "(memory_content reset + last_session_id wiped)",
diff --git a/server/services/proxy/service.py b/server/services/proxy/service.py
index 4f7ee7d4..8c68de46 100644
--- a/server/services/proxy/service.py
+++ b/server/services/proxy/service.py
@@ -5,6 +5,7 @@ scoring, geo-targeting, sticky sessions, failover, and cost tracking.
Follows the CompactionService singleton pattern.
"""
+import asyncio
import json
import time
from collections import deque
@@ -134,6 +135,11 @@ class ProxyService:
self._settings = settings
self._providers: Dict[str, _ProviderRuntime] = {}
self._routing_rules: List[RoutingRule] = []
+ # Serialize fetch + async credential/build + pointer swap. Atomic
+ # assignment prevents partial snapshots, while these locks prevent an
+ # older slow reload from publishing after a newer one.
+ self._provider_reload_lock = asyncio.Lock()
+ self._routing_reload_lock = asyncio.Lock()
self._daily_spend_usd: float = 0.0
self._initialized = False
@@ -145,11 +151,10 @@ class ProxyService:
"""
try:
# Load provider configs from database
- db_providers = await self._database.get_proxy_providers()
- await self._load_providers(db_providers)
+ await self.reload_providers()
# Load routing rules from database
- await self._reload_routing_rules()
+ await self.reload_routing_rules()
self._initialized = True
provider_names = [p.config.name for p in self._providers.values() if p.config.enabled]
@@ -286,14 +291,14 @@ class ProxyService:
"""Save a routing rule to database and refresh in-memory list."""
success = await self._database.save_proxy_routing_rule(rule_data)
if success:
- await self._reload_routing_rules()
+ await self.reload_routing_rules()
return success
async def delete_routing_rule(self, rule_id: int) -> bool:
"""Delete a routing rule and refresh in-memory list."""
success = await self._database.delete_proxy_routing_rule(rule_id)
if success:
- await self._reload_routing_rules()
+ await self.reload_routing_rules()
return success
async def test_provider(self, provider_name: str) -> Dict[str, Any]:
@@ -357,13 +362,25 @@ class ProxyService:
async def reload_providers(self) -> None:
"""Reload provider configs from database (called after tool/UI changes)."""
- db_providers = await self._database.get_proxy_providers()
- await self._load_providers(db_providers)
+ async with self._provider_reload_lock:
+ db_providers = await self._database.get_proxy_providers()
+ await self._load_providers(db_providers)
logger.info("Proxy providers reloaded", count=len(self._providers))
+ async def reload_routing_rules(self) -> None:
+ """Reload routing rules after an out-of-service database write."""
+ async with self._routing_reload_lock:
+ await self._reload_routing_rules()
+ logger.info("Proxy routing rules reloaded", count=len(self._routing_rules))
+
async def _load_providers(self, db_providers: list) -> None:
- """Build _ProviderRuntime instances from database rows."""
- self._providers.clear()
+ """Build and atomically publish a complete provider snapshot.
+
+ Credential lookups await. Mutating ``self._providers`` in place used
+ to expose an empty or half-built routing table to requests arriving
+ during a reload.
+ """
+ providers: Dict[str, _ProviderRuntime] = {}
for row in db_providers:
template = json.loads(row.get("url_template", "{}"))
provider = TemplateProxyProvider(name=row["name"], template=template)
@@ -395,7 +412,12 @@ class ProxyService:
else:
logger.warning("No credentials for proxy provider", name=config.name)
- self._providers[config.name] = runtime
+ providers[config.name] = runtime
+
+ # A dict reference assignment is atomic for readers in this event
+ # loop; every request now observes either the old complete snapshot
+ # or the new complete snapshot.
+ self._providers = providers
# ---- Internal helpers ----
@@ -438,11 +460,11 @@ class ProxyService:
return candidates[0]
async def _reload_routing_rules(self) -> None:
- """Reload routing rules from database."""
+ """Reload and atomically publish a complete routing-rule snapshot."""
db_rules = await self._database.get_proxy_routing_rules()
- self._routing_rules = []
+ routing_rules: List[RoutingRule] = []
for row in db_rules:
- self._routing_rules.append(
+ routing_rules.append(
RoutingRule(
id=row.get("id"),
domain_pattern=row["domain_pattern"],
@@ -456,7 +478,8 @@ class ProxyService:
priority=row.get("priority", 0),
)
)
- self._routing_rules.sort(key=lambda r: r.priority)
+ routing_rules.sort(key=lambda r: r.priority)
+ self._routing_rules = routing_rules
# ---- Singleton accessor (CompactionService pattern) ----
diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.py
index f5ddf2eb..41713562 100644
--- a/server/services/temporal/agent_activities.py
+++ b/server/services/temporal/agent_activities.py
@@ -243,23 +243,32 @@ async def persist_agent_turn(payload: Dict[str, Any]) -> Dict[str, Any]:
return {"appended": False, "trimmed_count": 0}
from core.container import container
- from services.memory.markdown import (
- append_to_memory_markdown,
- trim_markdown_window,
- )
+ from services.memory.runtime import append_memory_turns_atomic
database = container.database()
- params = await database.get_node_parameters(memory_node_id) or {}
- current = params.get("memory_content", "")
-
- updated = append_to_memory_markdown(current, "human", payload.get("human_text", ""))
- updated = append_to_memory_markdown(updated, "ai", payload.get("assistant_text", ""))
-
- window_size = int(payload.get("window_size", 10))
- trimmed_content, trimmed_pairs = trim_markdown_window(updated, window_size)
+ mutation_id = payload.get("mutation_id")
+ if not mutation_id:
+ # Temporal keeps activity_id stable across automatic retries, making
+ # it a durable idempotency key without changing workflow payloads.
+ try:
+ info = activity.info()
+ mutation_id = (
+ f"temporal-memory:{info.workflow_id}:{info.activity_id}:"
+ f"{memory_node_id}"
+ )
+ except Exception: # pragma: no cover - direct unit invocation
+ mutation_id = None
- params["memory_content"] = trimmed_content
- await database.save_node_parameters(memory_node_id, params)
+ params, trimmed_pairs, applied = await append_memory_turns_atomic(
+ database,
+ memory_node_id,
+ [
+ ("human", payload.get("human_text", "")),
+ ("ai", payload.get("assistant_text", "")),
+ ],
+ window_size=int(payload.get("window_size", 10)),
+ mutation_id=mutation_id,
+ )
# Broadcast so the parameter panel auto-refetches mid-run.
# CloudEvents v1.0 envelope (RFC §6.4) — type is
@@ -271,13 +280,18 @@ async def persist_agent_turn(payload: Dict[str, Any]) -> Dict[str, Any]:
from services.status_broadcaster import get_status_broadcaster
broadcaster = get_status_broadcaster()
- await broadcaster.broadcast_node_parameters_updated(
- memory_node_id,
- parameters=params,
- source_hint="agent",
- )
+ if applied:
+ await broadcaster.broadcast_node_parameters_updated(
+ memory_node_id,
+ parameters=params,
+ source_hint="agent",
+ )
- return {"appended": True, "trimmed_count": len(trimmed_pairs)}
+ return {
+ "appended": applied,
+ "applied": applied,
+ "trimmed_count": len(trimmed_pairs),
+ }
@activity.defn(name="agent.compact_memory.v1")