Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(db): bound migration lock waits, and run the deploy job through boot's path

Petr Bauched8208209f10
diff --git a/core/pyproject.toml b/core/pyproject.toml
index 013a0f0cc..30a40c42f 100644
--- a/core/pyproject.toml
+++ b/core/pyproject.toml
@@ -50,6 +50,7 @@ Issues = "https://github.com/sandbox-quantum/switch/issues"
[project.scripts]
switch-core = "switch_core.main:main"
+switch-migrate = "switch_core.main:migrate"
[build-system]
requires = ["hatchling"]
diff --git a/core/switch_core/config.py b/core/switch_core/config.py
index 7ea62112d..ab01ca1f4 100644
--- a/core/switch_core/config.py
+++ b/core/switch_core/config.py
@@ -229,6 +229,20 @@ class SwitchConfig(BaseSettings):
# never killed mid-transaction.
db_idle_in_transaction_session_timeout: str | None = None
+ # How long a migration waits for a lock before giving up (a Postgres
+ # interval such as "10s"; "0" waits forever). Migrations are DDL, so nearly
+ # every statement wants ACCESS EXCLUSIVE, and a request for one queues
+ # behind whatever transaction currently holds the table — and every reader
+ # arriving after it queues behind the request. A migration that waits out a
+ # long-running transaction therefore does not merely take longer; it stops
+ # the deployment still serving traffic beside it for as long as it waits.
+ # Failing instead turns that into an upgrade that did not happen, on a
+ # deployment that is still up on the old schema, which is the better of the
+ # two outcomes and the one worth retrying. Applies to the migration
+ # connection only: the application engine's statements take ordinary row
+ # and table locks that no amount of waiting escalates into this.
+ db_migration_lock_timeout: str = "10s"
+
# A Postgres server that goes away without closing its sockets — a managed
# instance failing over to its standby — leaves every connection open and
# apparently healthy. Nothing above the socket can tell the difference:
@@ -325,6 +339,16 @@ class SwitchConfig(BaseSettings):
)
return self
+ @model_validator(mode="after")
+ def _validate_migration_lock_timeout(self) -> "SwitchConfig":
+ if not _PG_INTERVAL_RE.match(self.db_migration_lock_timeout):
+ raise ValueError(
+ "DB_MIGRATION_LOCK_TIMEOUT must be a Postgres interval such "
+ "as '10s', '500ms' or a bare count of milliseconds ('0' waits "
+ f"forever), got {self.db_migration_lock_timeout!r}."
+ )
+ return self
+
@model_validator(mode="after")
def _validate_db_ssl_mode(self) -> "SwitchConfig":
allowed = {
diff --git a/core/switch_core/db/engine.py b/core/switch_core/db/engine.py
index 17f552e42..3e760eaf8 100644
--- a/core/switch_core/db/engine.py
+++ b/core/switch_core/db/engine.py
@@ -122,6 +122,27 @@ def app_connect_args(config: SwitchConfig) -> dict[str, object]:
return connect_args
+def migration_connect_args(config: SwitchConfig) -> dict[str, object]:
+ """asyncpg connect args for the engine Alembic runs migrations on.
+
+ The counterpart to `app_connect_args`: both start from
+ `config.db_connect_args` and each adds the one server setting that is right
+ for its side only. `lock_timeout` bounds how long a DDL statement waits for
+ the lock it needs, which the application engine must not have — a request
+ that waits a little for a busy row is normal there, and a query killed
+ because it did is a user-visible error for no gain.
+
+ Set on the connection rather than issued as a statement inside the
+ migration because `migrations/env.py` runs the whole upgrade in one
+ transaction: a `SET` there would be scoped to that transaction, which is
+ fine, but it would also have to be re-issued by every other path that opens
+ a migration connection. The connection is the one place all of them share.
+ """
+ connect_args = dict(config.db_connect_args)
+ connect_args["server_settings"] = {"lock_timeout": config.db_migration_lock_timeout}
+ return connect_args
+
+
def create_engine_from_config(
config: SwitchConfig, **engine_kwargs: object
) -> AsyncEngine:
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index 321eafc47..1c0423e2f 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -1079,6 +1079,33 @@ async def _migrate_and_grant(config: SwitchConfig) -> None:
)
+def migrate() -> None:
+ """Entry point for migrating a deployment without starting a server.
+
+ Exists so a deploy can bring the schema up in a job of its own, ahead of
+ the pods that will serve on it, and still run *this* code rather than a
+ bare `alembic upgrade head`. The difference is everything `_migrate_and_grant`
+ adds around the upgrade: the boot lock, so the job and a replica that
+ starts while it is running cannot both be applying DDL, and the grant
+ re-issue, without which a table the migration just created is invisible to
+ the runtime role. A job that skipped either would leave the deployment in a
+ state the server then has to repair at boot, one replica at a time.
+
+ Boot runs the same function, and must keep doing so: nothing guarantees a
+ deployment has a migration job, and a developer running the server against
+ a fresh database has none. Running it twice is not wasted work — the second
+ pass finds no pending revisions and re-issues grants that are already
+ correct.
+ """
+ config = SwitchConfig()
+ running_version = switch_core_version()
+ configure_logging(config, running_version)
+
+ logger.info("Migrating switch-core %s", running_version or "(version unknown)")
+
+ asyncio.run(_migrate_and_grant(config))
+
+
def main() -> None:
config = SwitchConfig()
running_version = switch_core_version()
diff --git a/core/switch_core/migrations/env.py b/core/switch_core/migrations/env.py
index a5f1b57ae..eea50e3cb 100644
--- a/core/switch_core/migrations/env.py
+++ b/core/switch_core/migrations/env.py
@@ -10,6 +10,7 @@ from switch_core.db import (
models as _models, # noqa: F401 — registers tables with Base.metadata
)
from switch_core.db.base import Base
+from switch_core.db.engine import migration_connect_args
from switch_core.logging_config import logging_is_configured
config = context.config
@@ -58,7 +59,7 @@ async def run_async_migrations() -> None:
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
- connect_args=switch_config.db_connect_args,
+ connect_args=migration_connect_args(switch_config),
)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
diff --git a/core/tests/switch_core/test_db_migration_lock_timeout.py b/core/tests/switch_core/test_db_migration_lock_timeout.py
new file mode 100644
index 000000000..83a8d0e93
--- /dev/null
+++ b/core/tests/switch_core/test_db_migration_lock_timeout.py
@@ -0,0 +1,84 @@
+"""`lock_timeout` on the connection migrations run over.
+
+A migration asks for ACCESS EXCLUSIVE on tables the running deployment is still
+reading, and a lock request that queues makes every reader behind it queue too.
+Bounding the wait turns "the deployment stalled for as long as one open
+transaction lasted" into "the upgrade failed and the old schema is still
+serving". It belongs on the migration connection and nowhere else: the
+application engine's own statements take locks that waiting is the correct
+response to.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from switch_core.config import SwitchConfig
+from switch_core.db.engine import app_connect_args, migration_connect_args
+
+_BASE_KWARGS = dict(
+ db_host="db",
+ db_port="5432",
+ db_user="postgres",
+ db_password="pw",
+ db_name="switch",
+ matrix_server_name="switch.local",
+ agent_registration_token="token",
+ jwt_secret_key="jwt",
+ gateway_admin_email="[email protected]",
+ gateway_admin_password="pw",
+)
+
+
+def _config(**overrides: object) -> SwitchConfig:
+ return SwitchConfig(**{**_BASE_KWARGS, **overrides}) # type: ignore[arg-type]
+
+
+class TestItIsOnByDefault:
+ def test_a_bound_is_configured_without_anyone_asking(self) -> None:
+ assert _config().db_migration_lock_timeout == "10s"
+
+ def test_it_reaches_asyncpg_as_a_server_setting(self) -> None:
+ assert migration_connect_args(_config()) == {
+ "server_settings": {"lock_timeout": "10s"}
+ }
+
+ def test_it_composes_with_tls(self) -> None:
+ config = _config(db_ssl_mode="require")
+
+ assert migration_connect_args(config) == {
+ "ssl": "require",
+ "server_settings": {"lock_timeout": "10s"},
+ }
+
+
+class TestItAppliesToMigrationsOnly:
+ def test_the_application_engine_never_sends_it(self) -> None:
+ # A query killed for waiting on a busy row is a user-visible error for
+ # no gain; only DDL escalates a wait into a queue behind it.
+ assert "server_settings" not in app_connect_args(_config())
+
+ def test_db_connect_args_stays_free_of_it(self) -> None:
+ # `boot_lock` and `main._prepare_database` build their connections from
+ # this property, and neither issues DDL that a bound would help.
+ assert _config().db_connect_args == {}
+
+
+class TestConfiguredValues:
+ @pytest.mark.parametrize("value", ["10s", "500ms", "30000", "2min", "0"])
+ def test_accepted_values(self, value: str) -> None:
+ assert _config(db_migration_lock_timeout=value).db_migration_lock_timeout == (
+ value
+ )
+
+ def test_zero_is_how_a_deployment_opts_out(self) -> None:
+ # Postgres reads lock_timeout=0 as "wait forever", so this is the
+ # pre-existing behaviour rather than a special case in our code.
+ assert migration_connect_args(_config(db_migration_lock_timeout="0")) == {
+ "server_settings": {"lock_timeout": "0"}
+ }
+
+ @pytest.mark.parametrize("value", ["", "soon", "10 seconds", "-5s", "10s;"])
+ def test_a_value_postgres_would_reject_fails_at_startup(self, value: str) -> None:
+ with pytest.raises(ValueError, match="DB_MIGRATION_LOCK_TIMEOUT"):
+ _config(db_migration_lock_timeout=value)
diff --git a/core/tests/test_migrate_entrypoint.py b/core/tests/test_migrate_entrypoint.py
new file mode 100644
index 000000000..5cda0a6e0
--- /dev/null
+++ b/core/tests/test_migrate_entrypoint.py
@@ -0,0 +1,57 @@
+"""The migration Job's command, and the console script behind it.
+
+The Helm pre-upgrade Job names a command as a string in a template no Python
+import ever touches, so a rename on either side breaks the deploy rather than
+the build — and breaks it at the point where a release is already half applied.
+These tie the two together.
+
+Why the Job runs `switch-migrate` rather than `alembic upgrade head` at all:
+the entry point wraps the upgrade in the boot advisory lock, so the Job and a
+restarting replica cannot both apply DDL, and reissues the runtime role's
+grants afterwards, without which the tables the migration just created are
+unreadable by the pods the release then rolls.
+"""
+
+from __future__ import annotations
+
+import re
+import tomllib
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+MIGRATE_JOB = (
+ REPO_ROOT / "deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml"
+)
+CORE_PYPROJECT = REPO_ROOT / "core/pyproject.toml"
+
+ENTRY_POINT = "switch-migrate"
+
+
+def _job_command() -> list[str]:
+ match = re.search(
+ r"^\s*command: (\[.*\])$", MIGRATE_JOB.read_text(), flags=re.MULTILINE
+ )
+ assert match is not None, f"no container command found in {MIGRATE_JOB}"
+ return re.findall(r'"([^"]+)"', match.group(1))
+
+
+def _console_scripts() -> dict[str, str]:
+ with CORE_PYPROJECT.open("rb") as handle:
+ return tomllib.load(handle)["project"]["scripts"]
+
+
+def test_the_job_runs_the_migration_entry_point() -> None:
+ assert _job_command() == [ENTRY_POINT]
+
+
+def test_the_entry_point_is_installed_by_the_package() -> None:
+ # The image installs the project, so anything declared here lands on PATH
+ # in the same container the Job runs — and anything not declared does not.
+ assert _console_scripts()[ENTRY_POINT] == "switch_core.main:migrate"
+
+
+def test_the_entry_point_resolves_to_a_callable() -> None:
+ module_path, _, attribute = _console_scripts()[ENTRY_POINT].partition(":")
+ module = __import__(module_path, fromlist=[attribute])
+
+ assert callable(getattr(module, attribute))
diff --git a/deploy/remote/helm/switch/templates/_helpers.tpl b/deploy/remote/helm/switch/templates/_helpers.tpl
index 10c96cc0b..eac995c80 100644
--- a/deploy/remote/helm/switch/templates/_helpers.tpl
+++ b/deploy/remote/helm/switch/templates/_helpers.tpl
@@ -517,6 +517,8 @@ Include with `nindent 12`.
- name: DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUT
value: {{ . | quote }}
{{- end }}
+- name: DB_MIGRATION_LOCK_TIMEOUT
+ value: {{ .Values.postgresql.migrationLockTimeout | quote }}
- name: AGENT_AUTH_CACHE_TTL_SECONDS
value: {{ .Values.switchCore.authCache.ttlSeconds | quote }}
- name: MATRIX_SERVER_NAME
diff --git a/deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml b/deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml
index 839a72ec0..c71a00f66 100644
--- a/deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml
+++ b/deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml
@@ -8,11 +8,17 @@
{{- $caOverride = printf "%s-db-ca-migrate" (include "switch.fullname" .) }}
{{- end }}
{{- $ctx := merge (dict "secretNameOverride" $secretOverride "dbCaConfigMapOverride" $caOverride) . }}
-# Runs `alembic upgrade head` before an upgrade rolls the new switch-core pods,
-# so a failed migration blocks the release instead of crash-looping the new
-# pods. pre-upgrade only: on a fresh install the managed Postgres does not exist
-# yet when pre-install hooks run, and switch-core self-migrates on startup
-# anyway (the same idempotent upgrade), so install is covered without a hook.
+# Migrates before an upgrade rolls the new switch-core pods, so a failed
+# migration blocks the release instead of crash-looping the new pods.
+# pre-upgrade only: on a fresh install the managed Postgres does not exist yet
+# when pre-install hooks run, and switch-core self-migrates on startup anyway
+# (the same idempotent upgrade), so install is covered without a hook.
+#
+# `switch-migrate` rather than `alembic upgrade head`: it is the same code the
+# server runs at boot, so the job takes the boot advisory lock — without which
+# it and a still-running old replica restarting could apply DDL at the same
+# time — and reissues the runtime role's grants, without which a table this
+# migration created is unreadable by the pods that follow it.
apiVersion: batch/v1
kind: Job
metadata:
@@ -42,7 +48,7 @@ spec:
- name: migrate
image: {{ include "switch.image" (dict "global" .Values.global "image" .Values.switchCore.image) }}
imagePullPolicy: {{ .Values.switchCore.imagePullPolicy }}
- command: ["alembic", "upgrade", "head"]
+ command: ["switch-migrate"]
env:
{{- include "switch.coreEnv" $ctx | nindent 12 }}
{{- if include "switch.dbCaBundleConfigMap" $ctx }}
diff --git a/deploy/remote/helm/switch/values.yaml b/deploy/remote/helm/switch/values.yaml
index 5fd3c4710..75fffe8dc 100644
--- a/deploy/remote/helm/switch/values.yaml
+++ b/deploy/remote/helm/switch/values.yaml
@@ -400,6 +400,16 @@ postgresql:
# in staging first. Migrations are unaffected; Alembic does not read it.
idleInTransactionSessionTimeout: ""
+ # How long a migration waits for a lock before failing (e.g. "10s"; "0" waits
+ # forever). Migrations are DDL, so they ask for ACCESS EXCLUSIVE on tables the
+ # release is still serving from, and that request queues behind whatever
+ # transaction holds the table — with every reader arriving after it queueing
+ # behind the request. Waiting therefore does not just delay the upgrade, it
+ # stalls the deployment beside it. Failing instead leaves the old schema up
+ # and the release un-applied, which is the outcome worth retrying. Raise it
+ # for a migration known to rewrite a large table while traffic is on it.
+ migrationLockTimeout: "10s"
+
# Password source. By default the password is read from the chart-managed
# Secret (secrets.postgresPassword). To source it from a pre-existing Secret
# instead (e.g. synced by external-secrets / sealed-secrets), set