Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
Enforce one recording per upload idempotency key (#476)
Murtaza Nasircde0d554a9b7
diff --git a/src/api/api_v1.py b/src/api/api_v1.pyindex 94591a03..31cdd1a9 100644--- a/src/api/api_v1.py+++ b/src/api/api_v1.py@@ -3834,16 +3834,30 @@ def upload_recording():key = (form.get('idempotency_key') or '').strip() or Noneif key is not None and len(key) > 100:return jsonify({'error': 'idempotency_key must be 1 to 100 characters', 'code': 'invalid_parameter'}), 400- if key:+ cutoff = datetime.utcnow() - timedelta(hours=24)++ def _replay():earlier = (Recording.query.filter(Recording.user_id == current_user.id, Recording.upload_idempotency_key == key,- Recording.created_at >= datetime.utcnow() - timedelta(hours=24))+ Recording.created_at >= cutoff).order_by(Recording.id.desc()).first())- if earlier is not None:- data = earlier.to_dict(viewer_user=current_user)- data['external_refs'] = [r.to_dict() for r in xr.refs_for(earlier.id, current_user.id)]- data['idempotent_replay'] = True- return jsonify(data), 200+ if earlier is None:+ return None+ data = earlier.to_dict(viewer_user=current_user)+ data['external_refs'] = [r.to_dict() for r in xr.refs_for(earlier.id, current_user.id)]+ data['idempotent_replay'] = True+ return jsonify(data), 200++ if key:+ replay = _replay()+ if replay is not None:+ return replay+ # The key is unique per user (#476): one older than 24 hours is+ # cleared so this upload can take it.+ Recording.query.filter(Recording.user_id == current_user.id, Recording.upload_idempotency_key == key,+ Recording.created_at < cutoff).update({'upload_idempotency_key': None},+ synchronize_session=False)+ db.session.commit()# Report tags and folders the upload cannot use (the pipeline drops them).def _usable(item):@@ -3878,7 +3892,7 @@ def upload_recording():xr.add(recording, current_user.id, clean)response = ingest_uploaded_recording(owner=current_user, uploaded_file=request.files.get('file'),- form=form, prepare=_prepare)+ form=form, prepare=_prepare, on_conflict=_replay if key else None)body, status = (response if isinstance(response, tuple) else (response, response.status_code))if status in (200, 201, 202):data = body.get_json() or {}diff --git a/src/api/recordings.py b/src/api/recordings.pyindex 7b75dddf..8b3c4528 100644--- a/src/api/recordings.py+++ b/src/api/recordings.py@@ -20,6 +20,7 @@ from flask_login import login_required, current_userfrom werkzeug.utils import secure_filenamefrom werkzeug.exceptions import RequestEntityTooLargefrom sqlalchemy import select+from sqlalchemy.exc import IntegrityErrorfrom email.utils import encode_rfc2231from urllib.parse import quote@@ -2224,12 +2225,16 @@ def ingest_uploaded_recording(success_status=202,reuse_duplicate=False,prepare=None,+ on_conflict=None,):"""Run an uploaded file through Speakr's standard ingestion pipeline.- prepare(recording), when given, runs before the recording is first- committed, so fields it sets (API v1 participants, references, the- idempotency key) arrive with the recording itself.+ prepare(recording), when given, runs before the file is stored and the+ recording is first committed, so fields it sets (API v1 participants,+ references, the idempotency key) arrive with the recording itself.+ on_conflict(), when given, answers an upload whose rows hit a unique+ index (a concurrent retry with the same idempotency key, #476) with a+ response, or None for the usual error."""try:if uploaded_file is None:@@ -2844,6 +2849,8 @@ def ingest_uploaded_recording()db.session.add(recording)db.session.flush() # Assign recording.id without committing transaction yet+ if prepare is not None:+ prepare(recording)storage_key = storage.build_recording_key(original_filename, recording.id, now=now)stored_object = storage.upload_local_file(@@ -2867,9 +2874,6 @@ def ingest_uploaded_recording()db.session.add(new_association)- if prepare is not None:- prepare(recording)-db.session.commit()if selected_tags:@@ -2932,6 +2936,18 @@ def ingest_uploaded_recording('effective_limit_mb': limit_mb,'audio_only_mode': audio_only_mode,}), 413+ except IntegrityError as e:+ # Another upload committed the same unique key first (#476): keep+ # nothing of this one and let the caller answer with that recording.+ db.session.rollback()+ if 'filepath' in locals() and filepath and os.path.exists(filepath):+ os.remove(filepath)+ replay = on_conflict() if on_conflict is not None else None+ if replay is not None:+ current_app.logger.info("Upload lost a race on a unique key; answering with the earlier recording")+ return replay+ current_app.logger.error(f"Error during file upload: {e}", exc_info=True)+ return jsonify({'error': 'An unexpected error occurred during upload.'}), 500except Exception as e:db.session.rollback()current_app.logger.error(f"Error during file upload: {e}", exc_info=True)diff --git a/src/init_db.py b/src/init_db.pyindex ef877e01..f943110a 100644--- a/src/init_db.py+++ b/src/init_db.py@@ -26,7 +26,7 @@ from src.utils import (migration_lock,run_once,)-from src.utils.database import ensure_migration_ledger+from src.utils.database import drop_index_if_exists, ensure_migration_ledger# ConfigurationENABLE_INQUIRE_MODE = os.environ.get('ENABLE_INQUIRE_MODE', 'false').lower() == 'true'@@ -189,6 +189,32 @@ def _migration_section(app, failures, name):)+def unique_upload_idempotency_key(app, engine):+ """One recording per upload idempotency key (#476).++ The first index was not unique, so overlapping retries could store a key+ twice: older duplicates lose their key (the newest keeps replaying), then+ the unique index replaces the old one. NULLs are distinct, so unkeyed+ recordings never clash.+ """+ if 'ux_recording_user_idem' not in {i['name'] for i in inspect(engine).get_indexes('recording')}:+ with engine.begin() as conn:+ dupes = conn.execute(text(+ 'SELECT user_id, upload_idempotency_key, MAX(id) FROM recording '+ 'WHERE upload_idempotency_key IS NOT NULL '+ 'GROUP BY user_id, upload_idempotency_key HAVING COUNT(*) > 1')).fetchall()+ for user_id, key, keep in dupes:+ conn.execute(text(+ 'UPDATE recording SET upload_idempotency_key = NULL '+ 'WHERE user_id = :u AND upload_idempotency_key = :k AND id < :keep'),+ {'u': user_id, 'k': key, 'keep': keep})+ if dupes:+ app.logger.info(f"Cleared {len(dupes)} duplicated upload idempotency keys")+ create_index_if_not_exists(engine, 'ux_recording_user_idem', 'recording',+ 'user_id, upload_idempotency_key', unique=True)+ drop_index_if_exists(engine, 'ix_recording_user_idem', 'recording')++def _run_migrations(app, engine):"""Apply every schema migration. Assumes the migration lock is held."""failures = []@@ -290,7 +316,7 @@ def _run_migrations(app, engine):# Idempotent API uploads (mailr spec G8).if add_column_if_not_exists(engine, 'recording', 'upload_idempotency_key', 'VARCHAR(100)'):app.logger.info("Added upload_idempotency_key column to recording table")- create_index_if_not_exists(engine, 'ix_recording_user_idem', 'recording', 'user_id, upload_idempotency_key')+ unique_upload_idempotency_key(app, engine)# Chunks are read and deleted by recording, in order (D#441)create_index_if_not_exists(engine, 'ix_transcript_chunk_recording', 'transcript_chunk','recording_id, chunk_index')diff --git a/src/models/recording.py b/src/models/recording.pyindex ba3f82b9..37d76bb7 100644--- a/src/models/recording.py+++ b/src/models/recording.py@@ -53,7 +53,8 @@ class Recording(db.Model):# listener in src/services/recording_changes.py, never by hand.updated_at = db.Column(db.DateTime, nullable=True, index=True)# API upload retries (mailr spec G8): a repeat with the same key by the same- # user within 24 hours returns this recording.+ # user within 24 hours returns this recording. Unique per user+ # (ux_recording_user_idem, #476); an expired key is cleared before reuse.upload_idempotency_key = db.Column(db.String(100), nullable=True)processing_time_seconds = db.Column(db.Integer, nullable=True)transcription_duration_seconds = db.Column(db.Integer, nullable=True) # Time taken for transcriptiondiff --git a/src/utils/database.py b/src/utils/database.pyindex d911954c..37e80b92 100644--- a/src/utils/database.py+++ b/src/utils/database.py@@ -322,6 +322,22 @@ def create_index_if_not_exists(engine, index_name, table_name, columns, unique=Freturn True+def drop_index_if_exists(engine, index_name, table_name):+ """Drop an index if it exists. Returns True if it was dropped."""+ inspector = inspect(engine)+ if table_name not in inspector.get_table_names():+ return False+ if index_name not in [idx['name'] for idx in inspector.get_indexes(table_name)]:+ return False+ with engine.connect() as conn:+ if engine.name == 'mysql':+ conn.execute(text(f'DROP INDEX {index_name} ON `{table_name}`'))+ else:+ conn.execute(text(f'DROP INDEX {index_name}'))+ conn.commit()+ return True++def drop_not_null(engine, table_name, column_name):"""Make an existing NOT NULL column nullable.diff --git a/tests/test_api_v1_external_refs.py b/tests/test_api_v1_external_refs.pyindex 9a8a8fd4..f062bf7e 100644--- a/tests/test_api_v1_external_refs.py+++ b/tests/test_api_v1_external_refs.py@@ -253,6 +253,9 @@ def test_an_old_key_does_not_replay(world):with _no_media_tools():second = _upload(world, idempotency_key="msg-old")assert second.status_code == 202 and second.get_json()["id"] != first.get_json()["id"]+ with app.app_context():+ keys = {r.id: r.upload_idempotency_key for r in Recording.query.filter_by(user_id=world["owner"])}+ assert keys[first.get_json()["id"]] is None and keys[second.get_json()["id"]] == "msg-old"def test_strict_and_lenient_tags(world):diff --git a/tests/test_upgrade_mailr_api.py b/tests/test_upgrade_mailr_api.pyindex bd81a093..de20c247 100644--- a/tests/test_upgrade_mailr_api.py+++ b/tests/test_upgrade_mailr_api.py@@ -152,7 +152,8 @@ def test_upload_idempotency_and_external_refs(tmp_path, fixture):_upgrade(db_path)con = sqlite3.connect(db_path)assert "upload_idempotency_key" in _columns(con, "recording")- assert "ix_recording_user_idem" in {r[1] for r in con.execute("PRAGMA index_list('recording')")}+ indexes = {r[1]: r[2] for r in con.execute("PRAGMA index_list('recording')")}+ assert indexes.get("ux_recording_user_idem") == 1 and "ix_recording_user_idem" not in indexesassert {"recording_id", "user_id", "system", "kind", "ref", "url", "label"} <= _columns(con, "recording_external_ref")con.close()diff --git a/tests/test_upload_idempotency_race.py b/tests/test_upload_idempotency_race.pynew file mode 100644index 00000000..6058214b--- /dev/null+++ b/tests/test_upload_idempotency_race.py@@ -0,0 +1,106 @@+"""One recording per upload idempotency key, also under overlapping retries (#476).++The endpoint checks the key long before the recording is written, so two+retries can both pass the check. The unique index decides: the loser answers+with the winner's recording and leaves nothing of its own behind.++SHARED-DB: uses the world fixture of test_api_v1_external_refs, which removes+its users, recordings, webhooks and stored media afterwards.+"""++import json+import os+import sys+from unittest.mock import patch++import pytest+from sqlalchemy import inspect, text+from sqlalchemy.exc import IntegrityError++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from src.app import app, db+from src.init_db import unique_upload_idempotency_key+from src.models import Recording, Webhook, WebhookDelivery+from src.services.storage import get_storage_service+from src.services.storage.service import StorageService+from tests.test_api_v1_external_refs import _no_media_tools, _upload, world # noqa: F401++CODEC = {"has_video": False, "audio_codec": "opus", "video_codec": None, "duration": 5.0}+++def test_a_retry_that_overlaps_the_first_upload_replays_it(world):+ with app.app_context():+ hook = Webhook(user_id=world["owner"], name="m", url="https://example.com/h", secret="s" * 40,+ events=json.dumps(["recording.created"]))+ db.session.add(hook)+ db.session.commit()+ hook_id = hook.id+ staging = get_storage_service().get_staging_dir()+ staged_before = set(os.listdir(staging))+ store = StorageService.upload_local_file+ winner = []++ def _codec(*args, **kwargs):+ # The first request has passed its key check; the retry runs to the end now.+ if not winner:+ winner.append(None)+ winner[0] = _upload(world, idempotency_key="msg-race")+ return CODEC++ with _no_media_tools() as enqueue, \+ patch("src.api.recordings.get_codec_info", side_effect=_codec), \+ patch.object(StorageService, "upload_local_file", autospec=True, side_effect=store) as stored:+ loser = _upload(world, idempotency_key="msg-race")+ assert enqueue.call_count == 1 and stored.call_count == 1++ first = winner[0]+ assert first.status_code == 202+ assert loser.status_code == 200 and loser.get_json()["idempotent_replay"] is True+ assert loser.get_json()["id"] == first.get_json()["id"]+ assert set(os.listdir(staging)) == staged_before+ with app.app_context():+ assert Recording.query.filter_by(user_id=world["owner"], original_filename="memo.webm").count() == 1+ assert WebhookDelivery.query.filter_by(webhook_id=hook_id).count() == 1+++def _index(engine, name):+ return next((i for i in inspect(engine).get_indexes("recording") if i["name"] == name), None)+++def test_the_migration_keeps_the_newest_of_duplicated_keys(world):+ with app.app_context():+ engine = db.engine+ try:+ # The schema before #476: a plain index that let a key in twice.+ with engine.begin() as conn:+ conn.execute(text("DROP INDEX ux_recording_user_idem"))+ conn.execute(text('CREATE INDEX ix_recording_user_idem ON "recording" '+ "(user_id, upload_idempotency_key)"))+ rows = [Recording(user_id=user, title="r", status="COMPLETED", original_filename="r.wav",+ upload_idempotency_key=key)+ for user, key in [("owner", "dup"), ("owner", "dup"), ("owner", "dup"),+ ("owner", "solo"), ("guest", "dup")]]+ for r, user in zip(rows, ["owner"] * 4 + ["guest"]):+ r.user_id = world[user]+ db.session.add_all(rows)+ db.session.flush()+ ids = [r.id for r in rows]+ db.session.commit() # an open session would block the DDL on PostgreSQL++ unique_upload_idempotency_key(app, engine)+ unique_upload_idempotency_key(app, engine) # second start: no-op++ db.session.expire_all()+ keys = [db.session.get(Recording, i).upload_idempotency_key for i in ids]+ assert keys == [None, None, "dup", "solo", "dup"]+ assert _index(engine, "ux_recording_user_idem")["unique"]+ assert _index(engine, "ix_recording_user_idem") is None+ db.session.add(Recording(user_id=world["owner"], title="r", status="COMPLETED",+ original_filename="r.wav", upload_idempotency_key="solo"))+ with pytest.raises(IntegrityError):+ db.session.commit()+ db.session.rollback()+ finally:+ db.session.rollback()+ unique_upload_idempotency_key(app, engine)