Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Update one tenant's template agents on one server at a time

Simon Flackdd780ab02d41
diff --git a/core/switch_core/db/stores/template_install_store.py b/core/switch_core/db/stores/template_install_store.py
index 42c900834..df523f93f 100644
--- a/core/switch_core/db/stores/template_install_store.py
+++ b/core/switch_core/db/stores/template_install_store.py
@@ -73,6 +73,22 @@ class TemplateInstallStore:
)
return list(result.scalars())
+ async def behind(
+ self, session: AsyncSession, template_id: str, revision: int
+ ) -> list[TemplateInstall]:
+ """The installs of a template still at a revision before `revision`,
+ oldest first: the ones an update has anything to do for."""
+ result = await session.execute(
+ select(TemplateInstall)
+ .where(
+ TemplateInstall.tenant_id == require_tenant_id(),
+ TemplateInstall.template_id == template_id,
+ TemplateInstall.revision < revision,
+ )
+ .order_by(TemplateInstall.created_at, TemplateInstall.id)
+ )
+ return list(result.scalars())
+
async def resources(
self, session: AsyncSession, install_id: str
) -> list[TemplateInstallResource]:
diff --git a/core/switch_core/management/template_updates.py b/core/switch_core/management/template_updates.py
index 36be6af04..e0924fa17 100644
--- a/core/switch_core/management/template_updates.py
+++ b/core/switch_core/management/template_updates.py
@@ -15,9 +15,9 @@ When anything was left as it is, the agent is left a notice (see
``template_status``): it is told, before the next message it receives, to
read how it differs from its template and tell its owner, who decides.
-Every write is conditional on what was read, so servers starting together
-each update an agent once at most: the one that loses a race leaves the
-agent to the one that won, and the install's revision moves only once.
+One server at a time works through a tenant, under a lock the others skip
+it for. Every write is also conditional on what was read, so even without
+the lock an agent is updated once at most and an install moves once.
The agents switch-buddy created before installs were recorded are recorded
first (``record_existing_buddies``), matched against the fingerprints of
@@ -28,12 +28,13 @@ from __future__ import annotations
import logging
import re
-from collections.abc import Sequence
+from collections.abc import AsyncIterator, Sequence
+from contextlib import asynccontextmanager
from typing import TYPE_CHECKING
-from sqlalchemy import select, update
+from sqlalchemy import func, select, update
from sqlalchemy.exc import IntegrityError
-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
from switch_core.db.models import (
Agent,
@@ -133,14 +134,29 @@ class TemplateUpdates:
async def run(self, tenant_ids: Sequence[str]) -> None:
"""Record old buddies and update every install, tenant by tenant. One
- tenant's failure is logged and the next is still updated."""
+ tenant's failure is logged and the next is still updated.
+
+ One server at a time works through a tenant: each takes the tenant's
+ lock first, and one that finds it held skips the tenant, which the
+ server holding it is already updating. The conditional writes would
+ keep two servers from updating anything twice anyway; the lock saves
+ them doing the same reads and renders, and keeps two servers from
+ each updating part of one install and neither moving it on."""
for tenant_id in tenant_ids:
try:
- await self.record_existing_buddies(tenant_id)
- for template in official_templates(
- self._agent_management, self._switch_buddy
- ):
- await self.update_installs(tenant_id, template)
+ async with self._tenant_lock(tenant_id) as held:
+ if not held:
+ logger.info(
+ "Another server is updating the agents of official "
+ "templates in tenant %s",
+ tenant_id,
+ )
+ continue
+ await self.record_existing_buddies(tenant_id)
+ for template in official_templates(
+ self._agent_management, self._switch_buddy
+ ):
+ await self.update_installs(tenant_id, template)
except Exception:
logger.error(
"Could not update the agents of official templates in tenant %s",
@@ -148,6 +164,33 @@ class TemplateUpdates:
exc_info=True,
)
+ @asynccontextmanager
+ async def _tenant_lock(self, tenant_id: str) -> AsyncIterator[bool]:
+ """A session-level advisory lock on the tenant's template updates,
+ held on one connection of its own for the pass and released after it
+ (or with the connection, if the process dies). Yields whether it was
+ taken.
+
+ The connection is in autocommit, so holding the lock does not hold a
+ transaction open for the length of the pass; and it is one connection
+ throughout, since a session-level lock is released only on the
+ connection that took it."""
+ key = func.hashtext(f"template-updates:{tenant_id}")
+ async with self._session_factory() as session:
+ engine = session.bind
+ if not isinstance(engine, AsyncEngine):
+ raise RuntimeError("template updates need sessions bound to an engine")
+ async with engine.connect() as connection:
+ locking = await connection.execution_options(isolation_level="AUTOCOMMIT")
+ held = bool(
+ (await locking.execute(select(func.pg_try_advisory_lock(key)))).scalar()
+ )
+ try:
+ yield held
+ finally:
+ if held:
+ await locking.execute(select(func.pg_advisory_unlock(key)))
+
# ── Buddies created before installs were recorded ───────────────────────
async def record_existing_buddies(self, tenant_id: str) -> int:
@@ -253,11 +296,7 @@ class TemplateUpdates:
async def update_installs(self, tenant_id: str, template: OfficialTemplate) -> None:
async with tenant_session(self._session_factory, tenant_id) as session:
- installs = [
- install
- for install in await self._store.of_template(session, template.id)
- if install.revision < template.version
- ]
+ installs = await self._store.behind(session, template.id, template.version)
for install in installs:
try:
await self.update_install(tenant_id, template, install)
diff --git a/core/tests/switch_core/management/test_template_updates.py b/core/tests/switch_core/management/test_template_updates.py
index f11cb3882..4eb12bb54 100644
--- a/core/tests/switch_core/management/test_template_updates.py
+++ b/core/tests/switch_core/management/test_template_updates.py
@@ -12,7 +12,7 @@ from typing import Any
import httpx
import pytest
-from sqlalchemy import select, update
+from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.bridges.agent.operations import context as op_context
@@ -434,3 +434,60 @@ async def _managed(harness: Harness, owner: User) -> list[dict[str, Any]]:
return await harness.management.service.list_managed_agents(
session, TENANT_ZERO_ID, owner.id
)
+
+
+async def _set_install_revision(harness: Harness, agent_id: str, revision: int) -> None:
+ install, _ = await _install(harness, agent_id)
+ async with harness.session_factory() as session:
+ await session.execute(
+ update(TemplateInstall)
+ .where(TemplateInstall.id == install.id)
+ .values(revision=revision)
+ )
+ await session.commit()
+
+
+async def test_a_tenant_another_server_is_updating_is_skipped(
+ harness: Harness, client: httpx.AsyncClient
+) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ agent_id = await _create_buddy(harness, client, owner, "switch-buddy-ada")
+ await _set_install_revision(harness, agent_id, 1)
+ key = func.hashtext(f"template-updates:{TENANT_ZERO_ID}")
+
+ async with harness.session_factory() as other_server:
+ assert (
+ await other_server.execute(select(func.pg_try_advisory_lock(key)))
+ ).scalar()
+ await _updates(harness).run([TENANT_ZERO_ID])
+ install, _ = await _install(harness, agent_id)
+ assert install.revision == 1
+ await other_server.execute(select(func.pg_advisory_unlock(key)))
+
+ await _updates(harness).run([TENANT_ZERO_ID])
+ install, _ = await _install(harness, agent_id)
+ assert install.revision == _buddy().version
+
+ # Released after the pass: the next server takes it.
+ async with harness.session_factory() as next_server:
+ assert (
+ await next_server.execute(select(func.pg_try_advisory_lock(key)))
+ ).scalar()
+ await next_server.execute(select(func.pg_advisory_unlock(key)))
+
+
+async def test_only_installs_behind_the_revision_are_read(
+ harness: Harness, client: httpx.AsyncClient
+) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ behind_id = await _create_buddy(harness, client, owner, "switch-buddy-ada")
+ current_id = await _create_buddy(harness, client, owner, "switch-buddy-ada-2")
+ await _set_install_revision(harness, behind_id, 1)
+
+ async with harness.session_factory() as session:
+ behind = await TemplateInstallStore().behind(
+ session, BUDDY_ID, _buddy().version
+ )
+
+ assert [i.id for i in behind] == [(await _install(harness, behind_id))[0].id]
+ assert (await _install(harness, current_id))[0].id not in [i.id for i in behind]