Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

test(core): fail, naming the holder, when a leaked transaction blocks the reset

Petr Bauch2d4dd5c9fff1
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml
index 3fac98b6c..b89465390 100644
--- a/.github/workflows/pr-ci.yml
+++ b/.github/workflows/pr-ci.yml
@@ -226,15 +226,15 @@ jobs:
# boots and owns a container of its own: no test shares a database with
# another running at the same time.
#
- # A test still running after five minutes has every thread's stack
- # dumped to the log, so a hang names itself.
+ # A test still running after five minutes fails (pytest-timeout), with
+ # its name and stack, and the rest of the shard carries on.
- name: pytest
run: >-
uv run --project core pytest -c core/pyproject.toml core/tests/
-n auto
--splits 3 --group ${{ matrix.group }}
--splitting-algorithm least_duration
- -o faulthandler_timeout=300
+ --timeout 300
console:
name: Switch Console
diff --git a/core/pyproject.toml b/core/pyproject.toml
index 338ebbb68..77ab5017b 100644
--- a/core/pyproject.toml
+++ b/core/pyproject.toml
@@ -71,6 +71,7 @@ dev = [
"testcontainers>=4.14.2",
"pytest-xdist>=3.6",
"pytest-split>=0.10",
+ "pytest-timeout>=2.3",
]
[tool.ruff]
diff --git a/core/tests/conftest.py b/core/tests/conftest.py
index 72aa32d6f..8f7ca3f97 100644
--- a/core/tests/conftest.py
+++ b/core/tests/conftest.py
@@ -10,6 +10,7 @@ import pytest
import pytest_asyncio
from sqlalchemy import insert, text
from sqlalchemy.engine import make_url
+from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import (
AsyncConnection,
AsyncEngine,
@@ -86,6 +87,19 @@ _TRUNCATE_ALL = (
)
+# Long enough for a slow machine, short enough to beat a CI job's limit.
+_TRUNCATE_LOCK_TIMEOUT = "60s"
+
+# What is still connected, for a TRUNCATE that could not take its locks.
+_OTHER_SESSIONS = text(
+ "SELECT pid, state, now() - xact_start AS in_transaction_for, "
+ "left(query, 300) AS last_query "
+ "FROM pg_stat_activity "
+ "WHERE datname = current_database() AND pid <> pg_backend_pid() "
+ "ORDER BY xact_start NULLS LAST"
+)
+
+
async def empty_the_database(engine: AsyncEngine) -> None:
"""Leave the schema in place and take every row out of it, tenant zero apart.
@@ -93,10 +107,29 @@ async def empty_the_database(engine: AsyncEngine) -> None:
building the schema from nothing is only one way to reach. Emptying it
instead is the same starting point for a fraction of the DDL, and it is
what every per-test fixture here uses.
+
+ TRUNCATE takes every table's exclusive lock, so a connection an earlier
+ test left in a transaction would make it wait forever. It waits a minute
+ instead, then fails naming the sessions still connected: the hang becomes
+ a failure that says who held the lock.
"""
- async with engine.begin() as conn:
- await conn.execute(text(_TRUNCATE_ALL))
- await _seed_tenant_zero(conn)
+ try:
+ async with engine.begin() as conn:
+ await conn.execute(
+ text(f"SET LOCAL lock_timeout = '{_TRUNCATE_LOCK_TIMEOUT}'")
+ )
+ await conn.execute(text(_TRUNCATE_ALL))
+ await _seed_tenant_zero(conn)
+ except DBAPIError as exc:
+ if getattr(exc.orig, "sqlstate", None) != "55P03": # lock_not_available
+ raise
+ async with engine.connect() as conn:
+ sessions = (await conn.execute(_OTHER_SESSIONS)).mappings().all()
+ raise AssertionError(
+ f"emptying the database waited {_TRUNCATE_LOCK_TIMEOUT} for its "
+ "locks; an earlier test left a connection holding them. Still "
+ "connected:\n" + "\n".join(str(dict(session)) for session in sessions)
+ ) from exc
@pytest.fixture(scope="session")
diff --git a/core/uv.lock b/core/uv.lock
index a63be3079..37a141f52 100644
--- a/core/uv.lock
+++ b/core/uv.lock
@@ -1979,6 +1979,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ae/a1/d4423657caaa8be9b31e491592b49cebdcfd434d3e74512ce71f6ec39905/pytest_split-0.11.0-py3-none-any.whl", hash = "sha256:899d7c0f5730da91e2daf283860eb73b503259cb416851a65599368849c7f382", size = 11911, upload-time = "2026-02-03T09:14:33.708Z" },
]
+[[package]]
+name = "pytest-timeout"
+version = "2.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" },
+]
+
[[package]]
name = "pytest-xdist"
version = "3.8.0"
@@ -2402,6 +2414,7 @@ dev = [
{ name = "pytest" },
{ name = "pytest-asyncio" },
{ name = "pytest-split" },
+ { name = "pytest-timeout" },
{ name = "pytest-xdist" },
{ name = "ruff" },
{ name = "testcontainers" },
@@ -2441,6 +2454,7 @@ dev = [
{ name = "pytest", specifier = ">=8" },
{ name = "pytest-asyncio", specifier = ">=0.24" },
{ name = "pytest-split", specifier = ">=0.10" },
+ { name = "pytest-timeout", specifier = ">=2.3" },
{ name = "pytest-xdist", specifier = ">=3.6" },
{ name = "ruff", specifier = ">=0.15" },
{ name = "testcontainers", specifier = ">=4.14.2" },