Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(db): bound migration lock waits, and run the deploy job through boot's path
Petr Bauched8208209f10
diff --git a/core/pyproject.toml b/core/pyproject.tomlindex 013a0f0cc..30a40c42f 100644--- a/core/pyproject.toml+++ b/core/pyproject.toml@@ -50,6 +50,7 @@ Issues = "https://github.com/sandbox-quantum/switch/issues"[project.scripts]switch-core = "switch_core.main:main"+switch-migrate = "switch_core.main:migrate"[build-system]requires = ["hatchling"]diff --git a/core/switch_core/config.py b/core/switch_core/config.pyindex 7ea62112d..ab01ca1f4 100644--- a/core/switch_core/config.py+++ b/core/switch_core/config.py@@ -229,6 +229,20 @@ class SwitchConfig(BaseSettings):# never killed mid-transaction.db_idle_in_transaction_session_timeout: str | None = None+ # How long a migration waits for a lock before giving up (a Postgres+ # interval such as "10s"; "0" waits forever). Migrations are DDL, so nearly+ # every statement wants ACCESS EXCLUSIVE, and a request for one queues+ # behind whatever transaction currently holds the table — and every reader+ # arriving after it queues behind the request. A migration that waits out a+ # long-running transaction therefore does not merely take longer; it stops+ # the deployment still serving traffic beside it for as long as it waits.+ # Failing instead turns that into an upgrade that did not happen, on a+ # deployment that is still up on the old schema, which is the better of the+ # two outcomes and the one worth retrying. Applies to the migration+ # connection only: the application engine's statements take ordinary row+ # and table locks that no amount of waiting escalates into this.+ db_migration_lock_timeout: str = "10s"+# A Postgres server that goes away without closing its sockets — a managed# instance failing over to its standby — leaves every connection open and# apparently healthy. Nothing above the socket can tell the difference:@@ -325,6 +339,16 @@ class SwitchConfig(BaseSettings):)return self+ @model_validator(mode="after")+ def _validate_migration_lock_timeout(self) -> "SwitchConfig":+ if not _PG_INTERVAL_RE.match(self.db_migration_lock_timeout):+ raise ValueError(+ "DB_MIGRATION_LOCK_TIMEOUT must be a Postgres interval such "+ "as '10s', '500ms' or a bare count of milliseconds ('0' waits "+ f"forever), got {self.db_migration_lock_timeout!r}."+ )+ return self+@model_validator(mode="after")def _validate_db_ssl_mode(self) -> "SwitchConfig":allowed = {diff --git a/core/switch_core/db/engine.py b/core/switch_core/db/engine.pyindex 17f552e42..3e760eaf8 100644--- a/core/switch_core/db/engine.py+++ b/core/switch_core/db/engine.py@@ -122,6 +122,27 @@ def app_connect_args(config: SwitchConfig) -> dict[str, object]:return connect_args+def migration_connect_args(config: SwitchConfig) -> dict[str, object]:+ """asyncpg connect args for the engine Alembic runs migrations on.++ The counterpart to `app_connect_args`: both start from+ `config.db_connect_args` and each adds the one server setting that is right+ for its side only. `lock_timeout` bounds how long a DDL statement waits for+ the lock it needs, which the application engine must not have — a request+ that waits a little for a busy row is normal there, and a query killed+ because it did is a user-visible error for no gain.++ Set on the connection rather than issued as a statement inside the+ migration because `migrations/env.py` runs the whole upgrade in one+ transaction: a `SET` there would be scoped to that transaction, which is+ fine, but it would also have to be re-issued by every other path that opens+ a migration connection. The connection is the one place all of them share.+ """+ connect_args = dict(config.db_connect_args)+ connect_args["server_settings"] = {"lock_timeout": config.db_migration_lock_timeout}+ return connect_args++def create_engine_from_config(config: SwitchConfig, **engine_kwargs: object) -> AsyncEngine:diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex 321eafc47..1c0423e2f 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -1079,6 +1079,33 @@ async def _migrate_and_grant(config: SwitchConfig) -> None:)+def migrate() -> None:+ """Entry point for migrating a deployment without starting a server.++ Exists so a deploy can bring the schema up in a job of its own, ahead of+ the pods that will serve on it, and still run *this* code rather than a+ bare `alembic upgrade head`. The difference is everything `_migrate_and_grant`+ adds around the upgrade: the boot lock, so the job and a replica that+ starts while it is running cannot both be applying DDL, and the grant+ re-issue, without which a table the migration just created is invisible to+ the runtime role. A job that skipped either would leave the deployment in a+ state the server then has to repair at boot, one replica at a time.++ Boot runs the same function, and must keep doing so: nothing guarantees a+ deployment has a migration job, and a developer running the server against+ a fresh database has none. Running it twice is not wasted work — the second+ pass finds no pending revisions and re-issues grants that are already+ correct.+ """+ config = SwitchConfig()+ running_version = switch_core_version()+ configure_logging(config, running_version)++ logger.info("Migrating switch-core %s", running_version or "(version unknown)")++ asyncio.run(_migrate_and_grant(config))++def main() -> None:config = SwitchConfig()running_version = switch_core_version()diff --git a/core/switch_core/migrations/env.py b/core/switch_core/migrations/env.pyindex a5f1b57ae..eea50e3cb 100644--- a/core/switch_core/migrations/env.py+++ b/core/switch_core/migrations/env.py@@ -10,6 +10,7 @@ from switch_core.db import (models as _models, # noqa: F401 — registers tables with Base.metadata)from switch_core.db.base import Base+from switch_core.db.engine import migration_connect_argsfrom switch_core.logging_config import logging_is_configuredconfig = context.config@@ -58,7 +59,7 @@ async def run_async_migrations() -> None:config.get_section(config.config_ini_section, {}),prefix="sqlalchemy.",poolclass=pool.NullPool,- connect_args=switch_config.db_connect_args,+ connect_args=migration_connect_args(switch_config),)async with connectable.connect() as connection:await connection.run_sync(do_run_migrations)diff --git a/core/tests/switch_core/test_db_migration_lock_timeout.py b/core/tests/switch_core/test_db_migration_lock_timeout.pynew file mode 100644index 000000000..83a8d0e93--- /dev/null+++ b/core/tests/switch_core/test_db_migration_lock_timeout.py@@ -0,0 +1,84 @@+"""`lock_timeout` on the connection migrations run over.++A migration asks for ACCESS EXCLUSIVE on tables the running deployment is still+reading, and a lock request that queues makes every reader behind it queue too.+Bounding the wait turns "the deployment stalled for as long as one open+transaction lasted" into "the upgrade failed and the old schema is still+serving". It belongs on the migration connection and nowhere else: the+application engine's own statements take locks that waiting is the correct+response to.+"""++from __future__ import annotations++import pytest++from switch_core.config import SwitchConfig+from switch_core.db.engine import app_connect_args, migration_connect_args++_BASE_KWARGS = dict(+ db_host="db",+ db_port="5432",+ db_user="postgres",+ db_password="pw",+ db_name="switch",+ matrix_server_name="switch.local",+ agent_registration_token="token",+ jwt_secret_key="jwt",+ gateway_admin_email="[email protected]",+ gateway_admin_password="pw",+)+++def _config(**overrides: object) -> SwitchConfig:+ return SwitchConfig(**{**_BASE_KWARGS, **overrides}) # type: ignore[arg-type]+++class TestItIsOnByDefault:+ def test_a_bound_is_configured_without_anyone_asking(self) -> None:+ assert _config().db_migration_lock_timeout == "10s"++ def test_it_reaches_asyncpg_as_a_server_setting(self) -> None:+ assert migration_connect_args(_config()) == {+ "server_settings": {"lock_timeout": "10s"}+ }++ def test_it_composes_with_tls(self) -> None:+ config = _config(db_ssl_mode="require")++ assert migration_connect_args(config) == {+ "ssl": "require",+ "server_settings": {"lock_timeout": "10s"},+ }+++class TestItAppliesToMigrationsOnly:+ def test_the_application_engine_never_sends_it(self) -> None:+ # A query killed for waiting on a busy row is a user-visible error for+ # no gain; only DDL escalates a wait into a queue behind it.+ assert "server_settings" not in app_connect_args(_config())++ def test_db_connect_args_stays_free_of_it(self) -> None:+ # `boot_lock` and `main._prepare_database` build their connections from+ # this property, and neither issues DDL that a bound would help.+ assert _config().db_connect_args == {}+++class TestConfiguredValues:+ @pytest.mark.parametrize("value", ["10s", "500ms", "30000", "2min", "0"])+ def test_accepted_values(self, value: str) -> None:+ assert _config(db_migration_lock_timeout=value).db_migration_lock_timeout == (+ value+ )++ def test_zero_is_how_a_deployment_opts_out(self) -> None:+ # Postgres reads lock_timeout=0 as "wait forever", so this is the+ # pre-existing behaviour rather than a special case in our code.+ assert migration_connect_args(_config(db_migration_lock_timeout="0")) == {+ "server_settings": {"lock_timeout": "0"}+ }++ @pytest.mark.parametrize("value", ["", "soon", "10 seconds", "-5s", "10s;"])+ def test_a_value_postgres_would_reject_fails_at_startup(self, value: str) -> None:+ with pytest.raises(ValueError, match="DB_MIGRATION_LOCK_TIMEOUT"):+ _config(db_migration_lock_timeout=value)diff --git a/core/tests/test_migrate_entrypoint.py b/core/tests/test_migrate_entrypoint.pynew file mode 100644index 000000000..5cda0a6e0--- /dev/null+++ b/core/tests/test_migrate_entrypoint.py@@ -0,0 +1,57 @@+"""The migration Job's command, and the console script behind it.++The Helm pre-upgrade Job names a command as a string in a template no Python+import ever touches, so a rename on either side breaks the deploy rather than+the build — and breaks it at the point where a release is already half applied.+These tie the two together.++Why the Job runs `switch-migrate` rather than `alembic upgrade head` at all:+the entry point wraps the upgrade in the boot advisory lock, so the Job and a+restarting replica cannot both apply DDL, and reissues the runtime role's+grants afterwards, without which the tables the migration just created are+unreadable by the pods the release then rolls.+"""++from __future__ import annotations++import re+import tomllib+from pathlib import Path++REPO_ROOT = Path(__file__).resolve().parents[2]+MIGRATE_JOB = (+ REPO_ROOT / "deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml"+)+CORE_PYPROJECT = REPO_ROOT / "core/pyproject.toml"++ENTRY_POINT = "switch-migrate"+++def _job_command() -> list[str]:+ match = re.search(+ r"^\s*command: (\[.*\])$", MIGRATE_JOB.read_text(), flags=re.MULTILINE+ )+ assert match is not None, f"no container command found in {MIGRATE_JOB}"+ return re.findall(r'"([^"]+)"', match.group(1))+++def _console_scripts() -> dict[str, str]:+ with CORE_PYPROJECT.open("rb") as handle:+ return tomllib.load(handle)["project"]["scripts"]+++def test_the_job_runs_the_migration_entry_point() -> None:+ assert _job_command() == [ENTRY_POINT]+++def test_the_entry_point_is_installed_by_the_package() -> None:+ # The image installs the project, so anything declared here lands on PATH+ # in the same container the Job runs — and anything not declared does not.+ assert _console_scripts()[ENTRY_POINT] == "switch_core.main:migrate"+++def test_the_entry_point_resolves_to_a_callable() -> None:+ module_path, _, attribute = _console_scripts()[ENTRY_POINT].partition(":")+ module = __import__(module_path, fromlist=[attribute])++ assert callable(getattr(module, attribute))diff --git a/deploy/remote/helm/switch/templates/_helpers.tpl b/deploy/remote/helm/switch/templates/_helpers.tplindex 10c96cc0b..eac995c80 100644--- a/deploy/remote/helm/switch/templates/_helpers.tpl+++ b/deploy/remote/helm/switch/templates/_helpers.tpl@@ -517,6 +517,8 @@ Include with `nindent 12`.- name: DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUTvalue: {{ . | quote }}{{- end }}+- name: DB_MIGRATION_LOCK_TIMEOUT+ value: {{ .Values.postgresql.migrationLockTimeout | quote }}- name: AGENT_AUTH_CACHE_TTL_SECONDSvalue: {{ .Values.switchCore.authCache.ttlSeconds | quote }}- name: MATRIX_SERVER_NAMEdiff --git a/deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml b/deploy/remote/helm/switch/templates/switch-core/migrate-job.yamlindex 839a72ec0..c71a00f66 100644--- a/deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml+++ b/deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml@@ -8,11 +8,17 @@{{- $caOverride = printf "%s-db-ca-migrate" (include "switch.fullname" .) }}{{- end }}{{- $ctx := merge (dict "secretNameOverride" $secretOverride "dbCaConfigMapOverride" $caOverride) . }}-# Runs `alembic upgrade head` before an upgrade rolls the new switch-core pods,-# so a failed migration blocks the release instead of crash-looping the new-# pods. pre-upgrade only: on a fresh install the managed Postgres does not exist-# yet when pre-install hooks run, and switch-core self-migrates on startup-# anyway (the same idempotent upgrade), so install is covered without a hook.+# Migrates before an upgrade rolls the new switch-core pods, so a failed+# migration blocks the release instead of crash-looping the new pods.+# pre-upgrade only: on a fresh install the managed Postgres does not exist yet+# when pre-install hooks run, and switch-core self-migrates on startup anyway+# (the same idempotent upgrade), so install is covered without a hook.+#+# `switch-migrate` rather than `alembic upgrade head`: it is the same code the+# server runs at boot, so the job takes the boot advisory lock — without which+# it and a still-running old replica restarting could apply DDL at the same+# time — and reissues the runtime role's grants, without which a table this+# migration created is unreadable by the pods that follow it.apiVersion: batch/v1kind: Jobmetadata:@@ -42,7 +48,7 @@ spec:- name: migrateimage: {{ include "switch.image" (dict "global" .Values.global "image" .Values.switchCore.image) }}imagePullPolicy: {{ .Values.switchCore.imagePullPolicy }}- command: ["alembic", "upgrade", "head"]+ command: ["switch-migrate"]env:{{- include "switch.coreEnv" $ctx | nindent 12 }}{{- if include "switch.dbCaBundleConfigMap" $ctx }}diff --git a/deploy/remote/helm/switch/values.yaml b/deploy/remote/helm/switch/values.yamlindex 5fd3c4710..75fffe8dc 100644--- a/deploy/remote/helm/switch/values.yaml+++ b/deploy/remote/helm/switch/values.yaml@@ -400,6 +400,16 @@ postgresql:# in staging first. Migrations are unaffected; Alembic does not read it.idleInTransactionSessionTimeout: ""+ # How long a migration waits for a lock before failing (e.g. "10s"; "0" waits+ # forever). Migrations are DDL, so they ask for ACCESS EXCLUSIVE on tables the+ # release is still serving from, and that request queues behind whatever+ # transaction holds the table — with every reader arriving after it queueing+ # behind the request. Waiting therefore does not just delay the upgrade, it+ # stalls the deployment beside it. Failing instead leaves the old schema up+ # and the release un-applied, which is the outcome worth retrying. Raise it+ # for a migration known to rewrite a large table while traffic is on it.+ migrationLockTimeout: "10s"+# Password source. By default the password is read from the chart-managed# Secret (secrets.postgresPassword). To source it from a pre-existing Secret# instead (e.g. synced by external-secrets / sealed-secrets), set