Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

fix: replace the destructive user-table rebuild in the password-nullable migration with an in-place column swap (#379)

Murtaza Nasir8dbce2b36590
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 271d3d4d..6d82d0c7 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -55,6 +55,60 @@ Please review the [CLA document](CLA.md) before submitting your contribution. Wh
- Test your changes before submitting
- Keep PRs focused on a single feature or fix
+### Database Migrations
+
+Speakr has no migration framework. Schema changes are hand-written in
+`src/init_db.py` and run on every startup, against SQLite (the shipped default) and
+PostgreSQL (common in larger deployments). Both have to keep working, and a migration
+that misbehaves runs on databases full of other people's recordings, so this area has
+stricter rules than the rest of the codebase.
+
+**Use the helpers in `src/utils/database.py`, never raw DDL.** Each one alters a single
+thing and handles the differences between the two backends, including boolean defaults,
+type names and quoting of reserved words such as `user`.
+
+| Need | Helper |
+|---|---|
+| Add a column | `add_column_if_not_exists()` |
+| Relax `NOT NULL` | `drop_not_null()` |
+| Change a column type | `migrate_column_type()` |
+| Add an index | `create_index_if_not_exists()` |
+
+**Never rebuild a table from a written-out schema.** A migration that does
+`CREATE TABLE x_new (...)`, copies the rows, drops the original and renames has frozen
+that column list at the moment it was written. Every column added afterwards is
+silently destroyed the day the migration runs. Change the one column you care about
+instead, which is what the helpers do.
+
+**Make it idempotent.** Migrations run several times per container start, some of them
+concurrently: from the entrypoint, from the admin user script, and once per worker
+process. A migration must recognise that its work is already done and return without
+acting.
+
+**Make failure recoverable.** pysqlite does not open a transaction for DDL, so a bare
+`CREATE TABLE` or `ALTER TABLE` commits immediately even inside a
+`with engine.connect()` block, and a failure later in the sequence leaves it behind
+permanently. If a migration writes more than one statement, drive the transaction
+yourself; `drop_not_null()` shows the pattern.
+
+**Add a column before anything reads it.** Migrations execute top to bottom in one
+function, so referencing a column that is added further down fails on precisely the
+upgrade path the migration exists for.
+
+Verify with:
+
+```bash
+python -m pytest tests/test_migration_compatibility.py tests/test_migration_drop_not_null.py -q
+
+# and against a real PostgreSQL, which the tests skip without a URL
+docker compose -f docker-compose.postgres.yml up -d postgres
+SPEAKR_TEST_POSTGRES_URL=postgresql://speakr:speakr@localhost:5432/speakr \
+ python -m pytest tests/test_migration_drop_not_null.py -q
+```
+
+A migration needs a test asserting on what a bad one would take away, meaning the other
+columns, the indexes and the row values, not merely that the intended change happened.
+
### Commit Message Guidelines
Follow the format used in the project:
diff --git a/src/init_db.py b/src/init_db.py
index 8edf80c0..5932932e 100644
--- a/src/init_db.py
+++ b/src/init_db.py
@@ -16,7 +16,7 @@ from sqlalchemy import text, inspect
from src.database import db
from src.models import Recording, TranscriptChunk, SystemSetting, User
from src.services.embeddings import process_recording_chunks
-from src.utils import add_column_if_not_exists, migrate_column_type, create_index_if_not_exists
+from src.utils import add_column_if_not_exists, migrate_column_type, create_index_if_not_exists, drop_not_null
# Configuration
ENABLE_INQUIRE_MODE = os.environ.get('ENABLE_INQUIRE_MODE', 'false').lower() == 'true'
@@ -53,6 +53,48 @@ def classify_embedding_identifier_state(current_identifier, stored_identifier, l
return stored_identifier, migrated_from_legacy, outcome
+def _remove_orphaned_user_new(engine, app):
+ """Drop the 'user_new' table stranded by the pre-0.10.5 password migration.
+
+ That migration rebuilt the user table from hand-written DDL. pysqlite does
+ not wrap DDL in a transaction, so when the row copy failed the CREATE had
+ already been committed; every later startup then aborted on 'table
+ user_new already exists' and the migration could never complete (#379).
+
+ The table is only ever dropped once the real one is present and holds at
+ least as many rows, so a database left mid-rebuild keeps its only copy of
+ the data and gets a warning instead.
+ """
+ if engine.name != 'sqlite':
+ return
+
+ try:
+ inspector = inspect(engine)
+ tables = inspector.get_table_names()
+ if 'user_new' not in tables:
+ return
+
+ with engine.connect() as conn:
+ orphaned_rows = conn.execute(text('SELECT COUNT(*) FROM user_new')).scalar()
+ live_rows = (
+ conn.execute(text('SELECT COUNT(*) FROM "user"')).scalar()
+ if 'user' in tables else 0
+ )
+
+ if live_rows and live_rows >= orphaned_rows:
+ conn.execute(text('DROP TABLE user_new'))
+ conn.commit()
+ app.logger.info("Removed orphaned user_new table left by an interrupted migration")
+ else:
+ app.logger.warning(
+ "Leaving orphaned user_new table in place: it holds %d row(s) but the "
+ "live user table holds %d. Inspect the database before removing it.",
+ orphaned_rows, live_rows
+ )
+ except Exception as e:
+ app.logger.warning(f"Could not clean up orphaned user_new table: {e}")
+
+
def initialize_database(app):
"""
Initialize database schema and run migrations.
@@ -109,76 +151,17 @@ def initialize_database(app):
if add_column_if_not_exists(engine, 'user', 'sso_subject', 'VARCHAR(255)'):
app.logger.info("Added sso_subject column to user table")
- # Make password column nullable for SSO users
+ # SSO users authenticate against their provider and never hold a local
+ # password, so the column has to accept NULL.
try:
- inspector = inspect(engine)
- if 'user' in inspector.get_table_names():
- if engine.name == 'sqlite':
- # SQLite doesn't support ALTER COLUMN, so we need to check and recreate
- with engine.connect() as conn:
- result = conn.execute(text("SELECT sql FROM sqlite_master WHERE type='table' AND name='user'"))
- schema = result.scalar()
-
- if schema and 'password VARCHAR(60) NOT NULL' in schema:
- app.logger.info("Migrating user table to make password nullable for SSO support...")
-
- conn.execute(text("""
- CREATE TABLE user_new (
- id INTEGER NOT NULL,
- username VARCHAR(20) NOT NULL,
- email VARCHAR(120) NOT NULL,
- password VARCHAR(60),
- is_admin BOOLEAN,
- can_share_publicly BOOLEAN,
- transcription_language VARCHAR(10),
- output_language VARCHAR(50),
- ui_language VARCHAR(10),
- summary_prompt TEXT,
- extract_events BOOLEAN,
- name VARCHAR(100),
- job_title VARCHAR(100),
- company VARCHAR(100),
- diarize BOOLEAN,
- sso_provider VARCHAR(100),
- sso_subject VARCHAR(255),
- PRIMARY KEY (id),
- UNIQUE (username),
- UNIQUE (email)
- )
- """))
- conn.execute(text("""
- INSERT INTO user_new
- SELECT id, username, email, password, is_admin, can_share_publicly,
- transcription_language, output_language, ui_language,
- summary_prompt, extract_events, name, job_title, company,
- diarize, sso_provider, sso_subject
- FROM user
- """))
- conn.execute(text("DROP TABLE user"))
- conn.execute(text("ALTER TABLE user_new RENAME TO user"))
- conn.execute(text('CREATE UNIQUE INDEX IF NOT EXISTS ix_user_sso_subject ON "user" (sso_subject)'))
- conn.commit()
- app.logger.info("Successfully made password column nullable for SSO support")
- else:
- app.logger.info("Password column is already nullable, skipping migration")
-
- elif engine.name == 'postgresql':
- # PostgreSQL supports ALTER COLUMN directly
- with engine.connect() as conn:
- result = conn.execute(text("""
- SELECT is_nullable FROM information_schema.columns
- WHERE table_name = 'user' AND column_name = 'password'
- """))
- row = result.fetchone()
- if row and row[0] == 'NO':
- conn.execute(text('ALTER TABLE "user" ALTER COLUMN password DROP NOT NULL'))
- conn.commit()
- app.logger.info("Made password column nullable for SSO support (PostgreSQL)")
- else:
- app.logger.info("Password column is already nullable, skipping migration")
+ if drop_not_null(engine, 'user', 'password'):
+ app.logger.info("Made password column nullable for SSO support")
except Exception as e:
app.logger.warning(f"Could not migrate password column to nullable (may cause issues with SSO): {e}")
+ _remove_orphaned_user_new(engine, app)
+
+
if add_column_if_not_exists(engine, 'recording', 'mime_type', 'VARCHAR(100)'):
app.logger.info("Added mime_type column to recording table")
if add_column_if_not_exists(engine, 'recording', 'audio_duration_seconds', 'FLOAT'):
diff --git a/src/utils/__init__.py b/src/utils/__init__.py
index bab564e0..27ed5121 100644
--- a/src/utils/__init__.py
+++ b/src/utils/__init__.py
@@ -28,7 +28,8 @@ from .security import (
from .database import (
add_column_if_not_exists,
migrate_column_type,
- create_index_if_not_exists
+ create_index_if_not_exists,
+ drop_not_null
)
from .token_auth import (
@@ -61,6 +62,7 @@ __all__ = [
'add_column_if_not_exists',
'migrate_column_type',
'create_index_if_not_exists',
+ 'drop_not_null',
# Token authentication
'extract_token_from_request',
'hash_token',
diff --git a/src/utils/database.py b/src/utils/database.py
index a0ac0a31..d70f8076 100644
--- a/src/utils/database.py
+++ b/src/utils/database.py
@@ -118,6 +118,127 @@ def create_index_if_not_exists(engine, index_name, table_name, columns, unique=F
return True
+def drop_not_null(engine, table_name, column_name):
+ """
+ Make an existing NOT NULL column nullable.
+
+ PostgreSQL and MySQL alter the constraint in place. SQLite cannot alter a
+ column constraint at all, so it swaps in a fresh column: ADD COLUMN always
+ produces a nullable column, so copying the values across and renaming
+ reaches the same end state.
+
+ The SQLite path deliberately does NOT rebuild the table. A rebuild has to
+ restate the whole schema in hand-written DDL, which silently drops every
+ column, index and constraint the DDL has fallen behind on; that is exactly
+ how the previous version of this migration came to destroy 27 columns of
+ user settings on any database it managed to run to completion (#379).
+
+ The SQLite statements run inside one explicit transaction because pysqlite
+ does not open one for DDL by itself. Without it a failure partway through
+ leaves the half-built column committed and the migration permanently
+ wedged, which is the other half of #379. BEGIN IMMEDIATE takes the write
+ lock up front so the several worker processes that run migrations at
+ startup cannot race, and the constraint is re-checked under that lock.
+
+ Note: SQLite refuses to drop a column that is indexed, unique or part of
+ the primary key, so this helper cannot be used on such a column.
+
+ Args:
+ engine: SQLAlchemy engine
+ table_name: Name of the table
+ column_name: Name of the column to make nullable
+
+ Returns:
+ bool: True if the column was made nullable, False if there was nothing
+ to do (table or column missing, or already nullable).
+ """
+ inspector = inspect(engine)
+
+ if table_name not in inspector.get_table_names():
+ return False
+
+ columns = {col['name']: col for col in inspector.get_columns(table_name)}
+ temp_col = f"{column_name}__nullable_tmp"
+
+ # Resume an attempt that died between DROP COLUMN and RENAME COLUMN: the
+ # data is all in the temporary column, so completing the rename is safe.
+ if engine.name == 'sqlite' and column_name not in columns and temp_col in columns:
+ with engine.connect() as conn:
+ conn.execute(text(
+ f'ALTER TABLE "{table_name}" RENAME COLUMN "{temp_col}" TO "{column_name}"'
+ ))
+ conn.commit()
+ return True
+
+ column = columns.get(column_name)
+ if column is None or column.get('nullable', True):
+ return False
+
+ column_type = column['type'].compile(engine.dialect)
+
+ if engine.name == 'postgresql':
+ with engine.connect() as conn:
+ conn.execute(text(
+ f'ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" DROP NOT NULL'
+ ))
+ conn.commit()
+ return True
+
+ if engine.name == 'mysql':
+ with engine.connect() as conn:
+ conn.execute(text(
+ f'ALTER TABLE `{table_name}` MODIFY COLUMN `{column_name}` {column_type} NULL'
+ ))
+ conn.commit()
+ return True
+
+ if engine.name != 'sqlite':
+ raise NotImplementedError(
+ f"drop_not_null() does not support the '{engine.name}' dialect"
+ )
+
+ raw_connection = engine.raw_connection()
+ try:
+ dbapi_connection = raw_connection.driver_connection
+ previous_isolation = dbapi_connection.isolation_level
+ # Hand transaction control to us so that BEGIN covers the DDL too.
+ dbapi_connection.isolation_level = None
+ cursor = dbapi_connection.cursor()
+ try:
+ cursor.execute('BEGIN IMMEDIATE')
+
+ # Re-check under the write lock; a concurrent worker may have
+ # completed the migration between our inspection and this point.
+ still_not_null = any(
+ row[1] == column_name and row[3] == 1
+ for row in cursor.execute(f'PRAGMA table_info("{table_name}")').fetchall()
+ )
+ if not still_not_null:
+ cursor.execute('ROLLBACK')
+ return False
+
+ if temp_col in columns:
+ cursor.execute(f'ALTER TABLE "{table_name}" DROP COLUMN "{temp_col}"')
+
+ cursor.execute(f'ALTER TABLE "{table_name}" ADD COLUMN "{temp_col}" {column_type}')
+ cursor.execute(f'UPDATE "{table_name}" SET "{temp_col}" = "{column_name}"')
+ cursor.execute(f'ALTER TABLE "{table_name}" DROP COLUMN "{column_name}"')
+ cursor.execute(
+ f'ALTER TABLE "{table_name}" RENAME COLUMN "{temp_col}" TO "{column_name}"'
+ )
+ cursor.execute('COMMIT')
+ except Exception:
+ cursor.execute('ROLLBACK')
+ raise
+ finally:
+ cursor.close()
+ dbapi_connection.isolation_level = previous_isolation
+ finally:
+ raw_connection.close()
+
+ return True
+
+
def migrate_column_type(engine, table_name, column_name, new_type, transform_sql=None):
"""
Migrate a column to a new type if it exists.
diff --git a/tests/test_migration_compatibility.py b/tests/test_migration_compatibility.py
index e8a43b52..d71cd2ca 100644
--- a/tests/test_migration_compatibility.py
+++ b/tests/test_migration_compatibility.py
@@ -229,22 +229,35 @@ class TestMigrationCompatibility(unittest.TestCase):
raw_pattern = r"conn\.execute\s*\(\s*text\s*\(['\"][^'\"]*CREATE\s+(?:UNIQUE\s+)?INDEX[^'\"]*ON\s+[\"'`]?user"
raw_matches = re.findall(raw_pattern, self.content, re.IGNORECASE)
- # Count uses of create_index_if_not_exists for user table
- utility_pattern = r"create_index_if_not_exists\s*\([^)]*['\"]user['\"]"
- utility_matches = re.findall(utility_pattern, self.content, re.IGNORECASE)
-
- # All index creation on user table should use the utility
- # (excluding table recreation scenarios which have their own quoting)
- if len(raw_matches) > 0:
- # Check if these are in table recreation blocks (acceptable)
- table_recreation_pattern = r"CREATE\s+TABLE\s+user_new"
- has_table_recreation = re.search(table_recreation_pattern, self.content, re.IGNORECASE)
-
- if not has_table_recreation or len(raw_matches) > 1:
- self.fail(
- f"Found {len(raw_matches)} raw CREATE INDEX statements on 'user' table. "
- f"Use create_index_if_not_exists() for proper quoting of reserved keywords."
- )
+ # All index creation on the user table must use the utility.
+ self.assertEqual(
+ len(raw_matches), 0,
+ f"Found {len(raw_matches)} raw CREATE INDEX statements on 'user' table. "
+ f"Use create_index_if_not_exists() for proper quoting of reserved keywords."
+ )
+
+ def test_no_table_rebuild_from_frozen_ddl(self):
+ """
+ Ensure no migration recreates a live table from a hand-written column list.
+
+ A migration that spells out a table's schema in a CREATE TABLE and then
+ copies rows into it freezes that schema at the moment it was written.
+ Every column added afterwards is silently dropped the day the migration
+ finally runs. This destroyed 27 columns of user settings in #379.
+
+ Change a column in place instead: add_column_if_not_exists() to add,
+ drop_not_null() to relax a constraint, migrate_column_type() to retype.
+ """
+ rebuild_pattern = r"CREATE\s+TABLE\s+(\w+_new)\b"
+ matches = re.findall(rebuild_pattern, self.content, re.IGNORECASE)
+
+ self.assertEqual(
+ matches, [],
+ f"Found table rebuild(s) from frozen DDL in init_db.py: {matches}. "
+ f"A hand-written CREATE TABLE goes stale the moment the next column "
+ f"is added, and the rebuild then drops it. Alter the column in place "
+ f"with the helpers in src/utils/database.py instead."
+ )
if __name__ == '__main__':
diff --git a/tests/test_migration_drop_not_null.py b/tests/test_migration_drop_not_null.py
new file mode 100644
index 00000000..8ddaf839
--- /dev/null
+++ b/tests/test_migration_drop_not_null.py
@@ -0,0 +1,296 @@
+"""Tests for the drop_not_null() migration helper.
+
+Regression cover for #379, where the previous password-nullable migration
+rebuilt the whole user table from hand-written DDL. That approach failed in two
+ways at once: it stranded a half-created table when the row copy failed, and on
+any database where it did run to completion it silently dropped every column
+the frozen DDL had fallen behind on.
+
+The replacement swaps a single column, so the tests below assert on what the
+rebuild destroyed: the other columns, the indexes and the row data.
+
+Set SPEAKR_TEST_POSTGRES_URL to also exercise the PostgreSQL path, e.g.
+ SPEAKR_TEST_POSTGRES_URL=postgresql://speakr:speakr@localhost:5433/speakr
+"""
+
+import os
+import sqlite3
+
+import pytest
+from sqlalchemy import create_engine, inspect, text
+
+from src.utils.database import drop_not_null
+
+
+LEGACY_USER_TABLE = """
+ CREATE TABLE user (
+ id INTEGER NOT NULL,
+ username VARCHAR(20) NOT NULL,
+ email VARCHAR(120) NOT NULL,
+ password VARCHAR(60) NOT NULL,
+ is_admin BOOLEAN,
+ monthly_token_budget INTEGER,
+ speaker_count_mode VARCHAR(10),
+ inquire_allow_notes BOOLEAN,
+ PRIMARY KEY (id),
+ UNIQUE (username),
+ UNIQUE (email)
+ )
+"""
+
+
+def sqlite_engine(tmp_path):
+ """A SQLite database shaped like a pre-SSO user table with later columns bolted on."""
+ db_path = tmp_path / "migration.db"
+ connection = sqlite3.connect(db_path)
+ connection.executescript(LEGACY_USER_TABLE)
+ connection.executescript(
+ """
+ CREATE INDEX ix_user_speaker_count_mode ON user (speaker_count_mode);
+ CREATE TABLE recording (id INTEGER PRIMARY KEY, user_id INTEGER REFERENCES user(id));
+ INSERT INTO user VALUES (1, 'admin', '[email protected]', 'bcrypt-hash-1', 1, 2000000, 'range', 1);
+ INSERT INTO user VALUES (2, 'other', '[email protected]', 'bcrypt-hash-2', 0, NULL, 'exact', 0);
+ INSERT INTO recording VALUES (10, 1);
+ """
+ )
+ connection.commit()
+ connection.close()
+ return create_engine(f"sqlite:///{db_path}")
+
+
+def _columns(engine, table="user"):
+ return [col["name"] for col in inspect(engine).get_columns(table)]
+
+
+def _indexes(engine, table="user"):
+ return sorted(idx["name"] for idx in inspect(engine).get_indexes(table))
+
+
+def _is_nullable(engine, table, column):
+ return next(
+ col["nullable"] for col in inspect(engine).get_columns(table) if col["name"] == column
+ )
+
+
+class TestSQLite:
+ def test_makes_column_nullable(self, sqlite_engine):
+ assert _is_nullable(sqlite_engine, "user", "password") is False
+
+ assert drop_not_null(sqlite_engine, "user", "password") is True
+
+ assert _is_nullable(sqlite_engine, "user", "password") is True
+
+ def test_preserves_every_other_column(self, sqlite_engine):
+ """The frozen-DDL rebuild dropped 27 columns of settings on real databases."""
+ before = _columns(sqlite_engine)
+
+ drop_not_null(sqlite_engine, "user", "password")
+
+ assert set(_columns(sqlite_engine)) == set(before)
+
+ def test_moves_the_column_to_the_end(self, sqlite_engine):
+ """Documented side effect: SQLite can only append, so the column moves.
+
+ Harmless here because every read goes through the ORM by name; the
+ codebase has no positional INSERT or SELECT * against the user table.
+ """
+ drop_not_null(sqlite_engine, "user", "password")
+
+ assert _columns(sqlite_engine)[-1] == "password"
+
+ def test_preserves_indexes(self, sqlite_engine):
+ before = _indexes(sqlite_engine)
+
+ drop_not_null(sqlite_engine, "user", "password")
+
+ assert _indexes(sqlite_engine) == before
+
+ def test_preserves_row_data(self, sqlite_engine):
+ query = text("SELECT id, username, password, monthly_token_budget, speaker_count_mode "
+ "FROM user ORDER BY id")
+ with sqlite_engine.connect() as conn:
+ before = conn.execute(query).fetchall()
+
+ drop_not_null(sqlite_engine, "user", "password")
+
+ with sqlite_engine.connect() as conn:
+ assert conn.execute(query).fetchall() == before
+
+ def test_accepts_null_password_afterwards(self, sqlite_engine):
+ """The whole point: an SSO user can be provisioned without a password."""
+ drop_not_null(sqlite_engine, "user", "password")
+
+ with sqlite_engine.connect() as conn:
+ conn.execute(text(
+ "INSERT INTO user (username, email, password) VALUES ('sso', '[email protected]', NULL)"
+ ))
+ conn.commit()
+ assert conn.execute(text("SELECT COUNT(*) FROM user WHERE password IS NULL")).scalar() == 1
+
+ def test_leaves_other_not_null_constraints_alone(self, sqlite_engine):
+ drop_not_null(sqlite_engine, "user", "password")
+
+ assert _is_nullable(sqlite_engine, "user", "username") is False
+
+ def test_is_idempotent(self, sqlite_engine):
+ """The old migration re-ran and re-failed on every single startup."""
+ assert drop_not_null(sqlite_engine, "user", "password") is True
+ assert drop_not_null(sqlite_engine, "user", "password") is False
+ assert drop_not_null(sqlite_engine, "user", "password") is False
+
+ def test_no_op_for_unknown_table_or_column(self, sqlite_engine):
+ assert drop_not_null(sqlite_engine, "nonexistent", "password") is False
+ assert drop_not_null(sqlite_engine, "user", "nonexistent") is False
+
+ def test_failure_leaves_no_partial_state(self, sqlite_engine):
+ """SQLite refuses to drop an indexed column; nothing may survive the attempt.
+
+ This is the property the original migration lacked. Its CREATE TABLE was
+ committed outside any transaction, so the failure was permanent.
+ """
+ with sqlite_engine.connect() as conn:
+ conn.execute(text("CREATE INDEX ix_user_password ON user (password)"))
+ conn.commit()
+ before = _columns(sqlite_engine)
+
+ with pytest.raises(Exception):
+ drop_not_null(sqlite_engine, "user", "password")
+
+ assert _columns(sqlite_engine) == before
+ assert _is_nullable(sqlite_engine, "user", "password") is False
+
+ def test_resumes_an_interrupted_rename(self, sqlite_engine):
+ """Recover a database killed between DROP COLUMN and RENAME COLUMN."""
+ with sqlite_engine.connect() as conn:
+ conn.execute(text('ALTER TABLE "user" ADD COLUMN "password__nullable_tmp" VARCHAR(60)'))
+ conn.execute(text('UPDATE "user" SET "password__nullable_tmp" = "password"'))
+ conn.execute(text('ALTER TABLE "user" DROP COLUMN "password"'))
+ conn.commit()
+ assert "password" not in _columns(sqlite_engine)
+
+ assert drop_not_null(sqlite_engine, "user", "password") is True
+
+ assert "password" in _columns(sqlite_engine)
+ assert "password__nullable_tmp" not in _columns(sqlite_engine)
+ with sqlite_engine.connect() as conn:
+ assert conn.execute(
+ text("SELECT password FROM user WHERE id = 1")
+ ).scalar() == "bcrypt-hash-1"
+
+
+class TestSQLiteIssue379:
+ """End-to-end cover for the exact database state reported in #379."""
+
+ def test_stale_user_new_does_not_block_the_migration(self, sqlite_engine):
+ with sqlite_engine.connect() as conn:
+ conn.execute(text("CREATE TABLE user_new (id INTEGER PRIMARY KEY, username VARCHAR(20))"))
+ conn.commit()
+ before = _columns(sqlite_engine)
+
+ assert drop_not_null(sqlite_engine, "user", "password") is True
+
+ assert _is_nullable(sqlite_engine, "user", "password") is True
+ assert set(_columns(sqlite_engine)) == set(before)
+
+ def test_orphaned_table_cleanup_is_guarded(self, sqlite_engine):
+ """Never drop user_new while it holds rows the live table does not."""
+ from src.init_db import _remove_orphaned_user_new
+
+ class _App:
+ class logger:
+ @staticmethod
+ def info(*args, **kwargs):
+ pass
+
+ @staticmethod
+ def warning(*args, **kwargs):
+ pass
+
+ with sqlite_engine.connect() as conn:
+ conn.execute(text("CREATE TABLE user_new (id INTEGER PRIMARY KEY)"))
+ conn.execute(text("INSERT INTO user_new VALUES (1), (2), (3)"))
+ conn.commit()
+
+ _remove_orphaned_user_new(sqlite_engine, _App)
+ assert "user_new" in inspect(sqlite_engine).get_table_names(), (
+ "user_new held more rows than user and must be preserved for inspection"
+ )
+
+ with sqlite_engine.connect() as conn:
+ conn.execute(text("DELETE FROM user_new WHERE id = 3"))
+ conn.commit()
+
+ _remove_orphaned_user_new(sqlite_engine, _App)
+ assert "user_new" not in inspect(sqlite_engine).get_table_names()
+
+
+POSTGRES_URL = os.environ.get("SPEAKR_TEST_POSTGRES_URL")
+
+
[email protected](not POSTGRES_URL, reason="SPEAKR_TEST_POSTGRES_URL is not set")
+class TestPostgreSQL:
+ """The production deployment runs PostgreSQL, so the native path needs cover too."""
+
+ @pytest.fixture
+ def pg_engine(self):
+ engine = create_engine(POSTGRES_URL)
+ with engine.connect() as conn:
+ conn.execute(text('DROP TABLE IF EXISTS migration_test_user'))
+ conn.execute(text("""
+ CREATE TABLE migration_test_user (
+ id SERIAL PRIMARY KEY,
+ username VARCHAR(20) NOT NULL UNIQUE,
+ password VARCHAR(60) NOT NULL,
+ monthly_token_budget INTEGER
+ )
+ """))
+ conn.execute(text(
+ "INSERT INTO migration_test_user (username, password, monthly_token_budget) "
+ "VALUES ('admin', 'bcrypt-hash-1', 2000000)"
+ ))
+ conn.commit()
+ yield engine
+ with engine.connect() as conn:
+ conn.execute(text('DROP TABLE IF EXISTS migration_test_user'))
+ conn.commit()
+ engine.dispose()
+
+ def test_makes_column_nullable(self, pg_engine):
+ assert _is_nullable(pg_engine, "migration_test_user", "password") is False
+
+ assert drop_not_null(pg_engine, "migration_test_user", "password") is True
+
+ assert _is_nullable(pg_engine, "migration_test_user", "password") is True
+
+ def test_preserves_columns_and_data(self, pg_engine):
+ before = _columns(pg_engine, "migration_test_user")
+
+ drop_not_null(pg_engine, "migration_test_user", "password")
+
+ assert _columns(pg_engine, "migration_test_user") == before
+ with pg_engine.connect() as conn:
+ assert conn.execute(text(
+ "SELECT password, monthly_token_budget FROM migration_test_user WHERE username = 'admin'"
+ )).fetchone() == ("bcrypt-hash-1", 2000000)
+
+ def test_is_idempotent(self, pg_engine):
+ assert drop_not_null(pg_engine, "migration_test_user", "password") is True
+ assert drop_not_null(pg_engine, "migration_test_user", "password") is False
+
+ def test_accepts_null_password_afterwards(self, pg_engine):
+ drop_not_null(pg_engine, "migration_test_user", "password")
+
+ with pg_engine.connect() as conn:
+ conn.execute(text(
+ "INSERT INTO migration_test_user (username, password) VALUES ('sso', NULL)"
+ ))
+ conn.commit()
+ assert conn.execute(text(
+ "SELECT COUNT(*) FROM migration_test_user WHERE password IS NULL"
+ )).scalar() == 1
+
+ def test_leaves_other_not_null_constraints_alone(self, pg_engine):
+ drop_not_null(pg_engine, "migration_test_user", "password")
+
+ assert _is_nullable(pg_engine, "migration_test_user", "username") is False