Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Update one tenant's template agents on one server at a time
Simon Flackdd780ab02d41
diff --git a/core/switch_core/db/stores/template_install_store.py b/core/switch_core/db/stores/template_install_store.pyindex 42c900834..df523f93f 100644--- a/core/switch_core/db/stores/template_install_store.py+++ b/core/switch_core/db/stores/template_install_store.py@@ -73,6 +73,22 @@ class TemplateInstallStore:)return list(result.scalars())+ async def behind(+ self, session: AsyncSession, template_id: str, revision: int+ ) -> list[TemplateInstall]:+ """The installs of a template still at a revision before `revision`,+ oldest first: the ones an update has anything to do for."""+ result = await session.execute(+ select(TemplateInstall)+ .where(+ TemplateInstall.tenant_id == require_tenant_id(),+ TemplateInstall.template_id == template_id,+ TemplateInstall.revision < revision,+ )+ .order_by(TemplateInstall.created_at, TemplateInstall.id)+ )+ return list(result.scalars())+async def resources(self, session: AsyncSession, install_id: str) -> list[TemplateInstallResource]:diff --git a/core/switch_core/management/template_updates.py b/core/switch_core/management/template_updates.pyindex 36be6af04..e0924fa17 100644--- a/core/switch_core/management/template_updates.py+++ b/core/switch_core/management/template_updates.py@@ -15,9 +15,9 @@ When anything was left as it is, the agent is left a notice (see``template_status``): it is told, before the next message it receives, toread how it differs from its template and tell its owner, who decides.-Every write is conditional on what was read, so servers starting together-each update an agent once at most: the one that loses a race leaves the-agent to the one that won, and the install's revision moves only once.+One server at a time works through a tenant, under a lock the others skip+it for. Every write is also conditional on what was read, so even without+the lock an agent is updated once at most and an install moves once.The agents switch-buddy created before installs were recorded are recordedfirst (``record_existing_buddies``), matched against the fingerprints of@@ -28,12 +28,13 @@ from __future__ import annotationsimport loggingimport re-from collections.abc import Sequence+from collections.abc import AsyncIterator, Sequence+from contextlib import asynccontextmanagerfrom typing import TYPE_CHECKING-from sqlalchemy import select, update+from sqlalchemy import func, select, updatefrom sqlalchemy.exc import IntegrityError-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmakerfrom switch_core.db.models import (Agent,@@ -133,14 +134,29 @@ class TemplateUpdates:async def run(self, tenant_ids: Sequence[str]) -> None:"""Record old buddies and update every install, tenant by tenant. One- tenant's failure is logged and the next is still updated."""+ tenant's failure is logged and the next is still updated.++ One server at a time works through a tenant: each takes the tenant's+ lock first, and one that finds it held skips the tenant, which the+ server holding it is already updating. The conditional writes would+ keep two servers from updating anything twice anyway; the lock saves+ them doing the same reads and renders, and keeps two servers from+ each updating part of one install and neither moving it on."""for tenant_id in tenant_ids:try:- await self.record_existing_buddies(tenant_id)- for template in official_templates(- self._agent_management, self._switch_buddy- ):- await self.update_installs(tenant_id, template)+ async with self._tenant_lock(tenant_id) as held:+ if not held:+ logger.info(+ "Another server is updating the agents of official "+ "templates in tenant %s",+ tenant_id,+ )+ continue+ await self.record_existing_buddies(tenant_id)+ for template in official_templates(+ self._agent_management, self._switch_buddy+ ):+ await self.update_installs(tenant_id, template)except Exception:logger.error("Could not update the agents of official templates in tenant %s",@@ -148,6 +164,33 @@ class TemplateUpdates:exc_info=True,)+ @asynccontextmanager+ async def _tenant_lock(self, tenant_id: str) -> AsyncIterator[bool]:+ """A session-level advisory lock on the tenant's template updates,+ held on one connection of its own for the pass and released after it+ (or with the connection, if the process dies). Yields whether it was+ taken.++ The connection is in autocommit, so holding the lock does not hold a+ transaction open for the length of the pass; and it is one connection+ throughout, since a session-level lock is released only on the+ connection that took it."""+ key = func.hashtext(f"template-updates:{tenant_id}")+ async with self._session_factory() as session:+ engine = session.bind+ if not isinstance(engine, AsyncEngine):+ raise RuntimeError("template updates need sessions bound to an engine")+ async with engine.connect() as connection:+ locking = await connection.execution_options(isolation_level="AUTOCOMMIT")+ held = bool(+ (await locking.execute(select(func.pg_try_advisory_lock(key)))).scalar()+ )+ try:+ yield held+ finally:+ if held:+ await locking.execute(select(func.pg_advisory_unlock(key)))+# ── Buddies created before installs were recorded ───────────────────────async def record_existing_buddies(self, tenant_id: str) -> int:@@ -253,11 +296,7 @@ class TemplateUpdates:async def update_installs(self, tenant_id: str, template: OfficialTemplate) -> None:async with tenant_session(self._session_factory, tenant_id) as session:- installs = [- install- for install in await self._store.of_template(session, template.id)- if install.revision < template.version- ]+ installs = await self._store.behind(session, template.id, template.version)for install in installs:try:await self.update_install(tenant_id, template, install)diff --git a/core/tests/switch_core/management/test_template_updates.py b/core/tests/switch_core/management/test_template_updates.pyindex f11cb3882..4eb12bb54 100644--- a/core/tests/switch_core/management/test_template_updates.py+++ b/core/tests/switch_core/management/test_template_updates.py@@ -12,7 +12,7 @@ from typing import Anyimport httpximport pytest-from sqlalchemy import select, update+from sqlalchemy import func, select, updatefrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.bridges.agent.operations import context as op_context@@ -434,3 +434,60 @@ async def _managed(harness: Harness, owner: User) -> list[dict[str, Any]]:return await harness.management.service.list_managed_agents(session, TENANT_ZERO_ID, owner.id)+++async def _set_install_revision(harness: Harness, agent_id: str, revision: int) -> None:+ install, _ = await _install(harness, agent_id)+ async with harness.session_factory() as session:+ await session.execute(+ update(TemplateInstall)+ .where(TemplateInstall.id == install.id)+ .values(revision=revision)+ )+ await session.commit()+++async def test_a_tenant_another_server_is_updating_is_skipped(+ harness: Harness, client: httpx.AsyncClient+) -> None:+ owner = await add_member(harness.session_factory, "ada")+ agent_id = await _create_buddy(harness, client, owner, "switch-buddy-ada")+ await _set_install_revision(harness, agent_id, 1)+ key = func.hashtext(f"template-updates:{TENANT_ZERO_ID}")++ async with harness.session_factory() as other_server:+ assert (+ await other_server.execute(select(func.pg_try_advisory_lock(key)))+ ).scalar()+ await _updates(harness).run([TENANT_ZERO_ID])+ install, _ = await _install(harness, agent_id)+ assert install.revision == 1+ await other_server.execute(select(func.pg_advisory_unlock(key)))++ await _updates(harness).run([TENANT_ZERO_ID])+ install, _ = await _install(harness, agent_id)+ assert install.revision == _buddy().version++ # Released after the pass: the next server takes it.+ async with harness.session_factory() as next_server:+ assert (+ await next_server.execute(select(func.pg_try_advisory_lock(key)))+ ).scalar()+ await next_server.execute(select(func.pg_advisory_unlock(key)))+++async def test_only_installs_behind_the_revision_are_read(+ harness: Harness, client: httpx.AsyncClient+) -> None:+ owner = await add_member(harness.session_factory, "ada")+ behind_id = await _create_buddy(harness, client, owner, "switch-buddy-ada")+ current_id = await _create_buddy(harness, client, owner, "switch-buddy-ada-2")+ await _set_install_revision(harness, behind_id, 1)++ async with harness.session_factory() as session:+ behind = await TemplateInstallStore().behind(+ session, BUDDY_ID, _buddy().version+ )++ assert [i.id for i in behind] == [(await _install(harness, behind_id))[0].id]+ assert (await _install(harness, current_id))[0].id not in [i.id for i in behind]