Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

⚡️(backend) add opt-in psycopg connection pool

briquet8659182f4177
diff --git a/CHANGELOG.md b/CHANGELOG.md
index eb7091b3..e295fc5e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,6 +30,7 @@ and this project adheres to
- ✨(backend) introduce a token exchange endpoint for iframe embeds
- ⚡️(backend) send recording notification emails from a Celery task
- ✨(backend) allow static feature flags without an analytics backend #1809
+- ⚡️(backend) add an opt-in psycopg connection pool
### Changed
diff --git a/docs/installation/kubernetes.md b/docs/installation/kubernetes.md
index 1f729e8a..f8fa1b84 100644
--- a/docs/installation/kubernetes.md
+++ b/docs/installation/kubernetes.md
@@ -318,12 +318,17 @@ These are the environmental options available on meet backend.
| DJANGO_SECRET_KEY | Secret key used for Django security | |
| DJANGO_SILENCED_SYSTEM_CHECKS | Silence Django system checks | [] |
| DJANGO_ALLOW_UNSECURE_USER_LISTING | Allow unsecure user listing | false |
-| DB_ENGINE | Database engine used | django.db.backends.postgresql_psycopg2 |
+| DB_ENGINE | Database engine used | django.db.backends.postgresql |
| DB_NAME | Name of the database | meet |
| DB_USER | User used to connect to database | dinum |
| DB_PASSWORD | Password used to connect to the database | pass |
| DB_HOST | Hostname of the database | localhost |
| DB_PORT | Port to connect to database | 5432 |
+| DB_PSYCOPG_POOL_ENABLED | Enable the psycopg connection pool | false |
+| DB_PSYCOPG_POOL_MIN_SIZE | Connections each pool keeps open | 1 |
+| DB_PSYCOPG_POOL_MAX_SIZE | Maximum connections per pool | 3 |
+| DB_PSYCOPG_POOL_TIMEOUT | Seconds a request waits for a free pool connection before failing | 30 |
+| DB_PSYCOPG_POOL_MAX_IDLE | Seconds an idle connection above the minimum stays open | 600 |
| STORAGES_STATICFILES_BACKEND | Static file serving engine | whitenoise.storage.CompressedManifestStaticFilesStorage |
| AWS_S3_ENDPOINT_URL | S3 host endpoint | |
| AWS_S3_ACCESS_KEY_ID | S3 access key | |
diff --git a/env.d/development/postgresql.dist b/env.d/development/postgresql.dist
index b1c3e338..fdb8c676 100644
--- a/env.d/development/postgresql.dist
+++ b/env.d/development/postgresql.dist
@@ -8,4 +8,5 @@ DB_HOST=postgresql
DB_NAME=meet
DB_USER=dinum
DB_PASSWORD=pass
-DB_PORT=5432
\ No newline at end of file
+DB_PORT=5432
+DB_PSYCOPG_POOL_ENABLED=True
diff --git a/src/backend/core/tests/files/test_api_files_create.py b/src/backend/core/tests/files/test_api_files_create.py
index f08e1b69..6fade97f 100644
--- a/src/backend/core/tests/files/test_api_files_create.py
+++ b/src/backend/core/tests/files/test_api_files_create.py
@@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor
from urllib.parse import parse_qs, urlparse
from uuid import uuid4
+from django.db import connection
from django.utils import timezone
import pytest
@@ -361,18 +362,21 @@ def test_api_files_create_file_race_condition():
"""
def create_item(title):
- user = factories.UserFactory()
- client = APIClient()
- client.force_login(user)
- return client.post(
- "/api/v1.0/files/",
- {
- "title": title,
- "type": FileTypeChoices.BACKGROUND_IMAGE,
- "filename": "my_file.png",
- },
- format="json",
- )
+ try:
+ user = factories.UserFactory()
+ client = APIClient()
+ client.force_login(user)
+ return client.post(
+ "/api/v1.0/files/",
+ {
+ "title": title,
+ "type": FileTypeChoices.BACKGROUND_IMAGE,
+ "filename": "my_file.png",
+ },
+ format="json",
+ )
+ finally:
+ connection.close()
with ThreadPoolExecutor(max_workers=2) as executor:
future1 = executor.submit(create_item, "my item 1")
diff --git a/src/backend/core/tests/files/test_api_files_upload_ended.py b/src/backend/core/tests/files/test_api_files_upload_ended.py
index a89c578c..cd3f7c16 100644
--- a/src/backend/core/tests/files/test_api_files_upload_ended.py
+++ b/src/backend/core/tests/files/test_api_files_upload_ended.py
@@ -5,6 +5,7 @@ from concurrent.futures import ThreadPoolExecutor
from io import BytesIO
from django.core.files.storage import default_storage
+from django.db import connection
import pytest
from rest_framework.test import APIClient
@@ -295,9 +296,12 @@ def test_api_file_upload_ended_concurrent_calls_are_serialized(settings):
default_storage.save(file.temporary_file_key, BytesIO(b"my prose"))
def call_upload_ended():
- client = APIClient()
- client.force_login(user)
- return client.post(f"/api/v1.0/files/{file.id!s}/upload-ended/")
+ try:
+ client = APIClient()
+ client.force_login(user)
+ return client.post(f"/api/v1.0/files/{file.id!s}/upload-ended/")
+ finally:
+ connection.close()
with ThreadPoolExecutor(max_workers=2) as executor:
futures = [
diff --git a/src/backend/core/tests/test_settings_database_pool.py b/src/backend/core/tests/test_settings_database_pool.py
new file mode 100644
index 00000000..7925d297
--- /dev/null
+++ b/src/backend/core/tests/test_settings_database_pool.py
@@ -0,0 +1,130 @@
+"""Unit tests for the opt-in psycopg connection pool settings."""
+
+# pylint: disable=protected-access
+
+from django.core.exceptions import ImproperlyConfigured
+
+import pytest
+
+from meet import settings
+
+
+def make_configuration(**overrides):
+ """Build a throwaway configuration with plain values and its own DATABASES."""
+ attributes = {
+ "DATABASES": {
+ "default": {
+ "ENGINE": "django.db.backends.postgresql",
+ "OPTIONS": {"sslmode": "require"},
+ }
+ },
+ "DB_PSYCOPG_POOL_ENABLED": True,
+ "DB_PSYCOPG_POOL_MIN_SIZE": 1,
+ "DB_PSYCOPG_POOL_MAX_SIZE": 3,
+ "DB_PSYCOPG_POOL_TIMEOUT": 30.0,
+ "DB_PSYCOPG_POOL_MAX_IDLE": 600.0,
+ **overrides,
+ }
+ return type("PoolConfiguration", (settings.Test,), attributes)
+
+
+def test_database_pool_disabled_leaves_options_untouched():
+ """Without the flag, no connection pool is configured."""
+ configuration = make_configuration(DB_PSYCOPG_POOL_ENABLED=False)
+
+ configuration._configure_database_pool()
+
+ assert configuration.DATABASES["default"]["OPTIONS"] == {"sslmode": "require"}
+ assert "CONN_HEALTH_CHECKS" not in configuration.DATABASES["default"]
+
+
+def test_database_pool_enabled_merges_pool_options():
+ """The pool and keepalive options are added next to the existing ones."""
+ configuration = make_configuration()
+
+ configuration._configure_database_pool()
+
+ assert configuration.DATABASES["default"]["OPTIONS"] == {
+ "sslmode": "require",
+ "keepalives_idle": 60,
+ "keepalives_interval": 10,
+ "keepalives_count": 5,
+ "pool": {"min_size": 1, "max_size": 3, "timeout": 30.0, "max_idle": 600.0},
+ }
+
+
+def test_database_pool_enabled_keeps_explicit_keepalives():
+ """Keepalive values already set, e.g. from DATABASE_URL, are not overridden."""
+ configuration = make_configuration(
+ DATABASES={
+ "default": {
+ "ENGINE": "django.db.backends.postgresql",
+ "OPTIONS": {"keepalives_idle": "30", "keepalives": "0"},
+ }
+ }
+ )
+
+ configuration._configure_database_pool()
+
+ options = configuration.DATABASES["default"]["OPTIONS"]
+ assert options["keepalives_idle"] == "30"
+ assert options["keepalives"] == "0"
+ assert options["keepalives_interval"] == 10
+
+
[email protected]("health_checks", [None, False])
+def test_database_pool_enabled_forces_health_checks(health_checks):
+ """Pooled connections are checked on checkout, even if DATABASE_URL said no."""
+ database = {"ENGINE": "django.db.backends.postgresql"}
+ if health_checks is not None:
+ database["CONN_HEALTH_CHECKS"] = health_checks
+ configuration = make_configuration(DATABASES={"default": database})
+
+ configuration._configure_database_pool()
+
+ assert configuration.DATABASES["default"]["CONN_HEALTH_CHECKS"] is True
+
+
+def test_database_pool_enabled_without_options():
+ """A database without OPTIONS, as built from DATABASE_URL, gets them created."""
+ configuration = make_configuration(
+ DATABASES={"default": {"ENGINE": "django.db.backends.postgresql_psycopg2"}}
+ )
+
+ configuration._configure_database_pool()
+
+ assert configuration.DATABASES["default"]["OPTIONS"]["pool"]["max_size"] == 3
+
+
+ "overrides,match",
+ [
+ (
+ {"DATABASES": {"default": {"ENGINE": "django.db.backends.sqlite3"}}},
+ "requires the django.db.backends.postgresql engine",
+ ),
+ (
+ {
+ "DATABASES": {
+ "default": {
+ "ENGINE": "django.db.backends.postgresql",
+ "CONN_MAX_AGE": 60,
+ }
+ }
+ },
+ "cannot be combined with persistent connections",
+ ),
+ (
+ {"DB_PSYCOPG_POOL_MIN_SIZE": 6},
+ "MIN_SIZE cannot be greater than",
+ ),
+ ({"DB_PSYCOPG_POOL_TIMEOUT": 0.0}, "must be positive"),
+ ({"DB_PSYCOPG_POOL_MAX_IDLE": -1.0}, "must be positive"),
+ ],
+)
+def test_database_pool_invalid_configuration(overrides, match):
+ """An invalid pool setup fails at boot."""
+ configuration = make_configuration(**overrides)
+
+ with pytest.raises(ImproperlyConfigured, match=match):
+ configuration._configure_database_pool()
diff --git a/src/backend/meet/settings.py b/src/backend/meet/settings.py
index 66b2bfc2..ced1df39 100755
--- a/src/backend/meet/settings.py
+++ b/src/backend/meet/settings.py
@@ -17,6 +17,7 @@ import warnings
from os import path
from socket import gethostbyname, gethostname
+from django.core.exceptions import ImproperlyConfigured
from django.utils.translation import gettext_lazy as _
import dj_database_url
@@ -139,7 +140,7 @@ class Base(Configuration):
)
else {
"ENGINE": values.Value(
- "django.db.backends.postgresql_psycopg2",
+ "django.db.backends.postgresql",
environ_name="DB_ENGINE",
environ_prefix=None,
),
@@ -154,6 +155,21 @@ class Base(Configuration):
"PORT": values.Value(5432, environ_name="DB_PORT", environ_prefix=None),
}
}
+ DB_PSYCOPG_POOL_ENABLED = values.BooleanValue(
+ False, environ_name="DB_PSYCOPG_POOL_ENABLED", environ_prefix=None
+ )
+ DB_PSYCOPG_POOL_MIN_SIZE = values.PositiveIntegerValue(
+ 1, environ_name="DB_PSYCOPG_POOL_MIN_SIZE", environ_prefix=None
+ )
+ DB_PSYCOPG_POOL_MAX_SIZE = values.PositiveIntegerValue(
+ 3, environ_name="DB_PSYCOPG_POOL_MAX_SIZE", environ_prefix=None
+ )
+ DB_PSYCOPG_POOL_TIMEOUT = values.FloatValue(
+ 30.0, environ_name="DB_PSYCOPG_POOL_TIMEOUT", environ_prefix=None
+ )
+ DB_PSYCOPG_POOL_MAX_IDLE = values.FloatValue(
+ 600.0, environ_name="DB_PSYCOPG_POOL_MAX_IDLE", environ_prefix=None
+ )
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
# Static files (CSS, JavaScript, Images)
@@ -1543,6 +1559,50 @@ class Base(Configuration):
f"'{cls.AUDIT_LOG_DATA_STREAM_NAMESPACE}' must not contain '-'."
)
+ @classmethod
+ def _configure_database_pool(cls):
+ """Inject the psycopg pool options into the default database when enabled."""
+ if not cls.DB_PSYCOPG_POOL_ENABLED:
+ return
+
+ database = cls.DATABASES["default"]
+ if database.get("ENGINE") not in (
+ "django.db.backends.postgresql",
+ "django.db.backends.postgresql_psycopg2",
+ ):
+ raise ImproperlyConfigured(
+ "DB_PSYCOPG_POOL_ENABLED requires the django.db.backends.postgresql "
+ f"engine, got {database.get('ENGINE')!r}."
+ )
+ if database.get("CONN_MAX_AGE"):
+ raise ImproperlyConfigured(
+ "DB_PSYCOPG_POOL_ENABLED cannot be combined with persistent "
+ "connections; set CONN_MAX_AGE to 0."
+ )
+ if cls.DB_PSYCOPG_POOL_MIN_SIZE > cls.DB_PSYCOPG_POOL_MAX_SIZE:
+ raise ImproperlyConfigured(
+ "DB_PSYCOPG_POOL_MIN_SIZE cannot be greater than "
+ "DB_PSYCOPG_POOL_MAX_SIZE."
+ )
+ if cls.DB_PSYCOPG_POOL_TIMEOUT <= 0 or cls.DB_PSYCOPG_POOL_MAX_IDLE <= 0:
+ raise ImproperlyConfigured(
+ "DB_PSYCOPG_POOL_TIMEOUT and DB_PSYCOPG_POOL_MAX_IDLE must be positive."
+ )
+
+ database["OPTIONS"] = {
+ "keepalives_idle": 60,
+ "keepalives_interval": 10,
+ "keepalives_count": 5,
+ **database.get("OPTIONS", {}),
+ "pool": {
+ "min_size": cls.DB_PSYCOPG_POOL_MIN_SIZE,
+ "max_size": cls.DB_PSYCOPG_POOL_MAX_SIZE,
+ "timeout": cls.DB_PSYCOPG_POOL_TIMEOUT,
+ "max_idle": cls.DB_PSYCOPG_POOL_MAX_IDLE,
+ },
+ }
+ database["CONN_HEALTH_CHECKS"] = True
+
@classmethod
def post_setup(cls):
"""Post setup configuration.
@@ -1558,6 +1618,7 @@ class Base(Configuration):
cls._check_recording_encoding_maps()
cls._check_audit_log_data_stream_namespace()
+ cls._configure_database_pool()
if cls.LIVEKIT_API_TIMEOUT_SECONDS < 1:
raise ValueError("LIVEKIT_API_TIMEOUT_SECONDS must be at least 1")
diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml
index 62c81669..e0880408 100644
--- a/src/backend/pyproject.toml
+++ b/src/backend/pyproject.toml
@@ -52,7 +52,7 @@ dependencies = [
"menshen-client==0.2.0",
"nested-multipart-parser==1.6.0",
"posthog==7.44.0",
- "psycopg[binary]==3.3.4",
+ "psycopg[binary,pool]==3.3.4",
"pydantic==2.13.4",
"PyJWT==2.14.0",
"python-frontmatter==1.3.0",
diff --git a/src/backend/uv.lock b/src/backend/uv.lock
index 55a36b69..88117651 100644
--- a/src/backend/uv.lock
+++ b/src/backend/uv.lock
@@ -1332,7 +1332,7 @@ dependencies = [
{ name = "nested-multipart-parser" },
{ name = "phonenumbers" },
{ name = "posthog" },
- { name = "psycopg", extra = ["binary"] },
+ { name = "psycopg", extra = ["binary", "pool"] },
{ name = "pydantic" },
{ name = "pyjwt" },
{ name = "python-frontmatter" },
@@ -1398,7 +1398,7 @@ requires-dist = [
{ name = "nested-multipart-parser", specifier = "==1.6.0" },
{ name = "phonenumbers", specifier = "==9.0.37" },
{ name = "posthog", specifier = "==7.44.0" },
- { name = "psycopg", extras = ["binary"], specifier = "==3.3.4" },
+ { name = "psycopg", extras = ["binary", "pool"], specifier = "==3.3.4" },
{ name = "pydantic", specifier = "==2.13.4" },
{ name = "pyjwt", specifier = "==2.14.0" },
{ name = "python-frontmatter", specifier = "==1.3.0" },
@@ -1889,6 +1889,9 @@ wheels = [
binary = [
{ name = "psycopg-binary", marker = "implementation_name != 'pypy'" },
]
+pool = [
+ { name = "psycopg-pool" },
+]
[[package]]
name = "psycopg-binary"
@@ -1919,6 +1922,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" },
]
+[[package]]
+name = "psycopg-pool"
+version = "3.3.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" },
+]
+
[[package]]
name = "ptyprocess"
version = "0.7.0"