Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

Enforce one recording per upload idempotency key (#476)

Murtaza Nasircde0d554a9b7
diff --git a/src/api/api_v1.py b/src/api/api_v1.py
index 94591a03..31cdd1a9 100644
--- a/src/api/api_v1.py
+++ b/src/api/api_v1.py
@@ -3834,16 +3834,30 @@ def upload_recording():
key = (form.get('idempotency_key') or '').strip() or None
if key is not None and len(key) > 100:
return jsonify({'error': 'idempotency_key must be 1 to 100 characters', 'code': 'invalid_parameter'}), 400
- if key:
+ cutoff = datetime.utcnow() - timedelta(hours=24)
+
+ def _replay():
earlier = (Recording.query
.filter(Recording.user_id == current_user.id, Recording.upload_idempotency_key == key,
- Recording.created_at >= datetime.utcnow() - timedelta(hours=24))
+ Recording.created_at >= cutoff)
.order_by(Recording.id.desc()).first())
- if earlier is not None:
- data = earlier.to_dict(viewer_user=current_user)
- data['external_refs'] = [r.to_dict() for r in xr.refs_for(earlier.id, current_user.id)]
- data['idempotent_replay'] = True
- return jsonify(data), 200
+ if earlier is None:
+ return None
+ data = earlier.to_dict(viewer_user=current_user)
+ data['external_refs'] = [r.to_dict() for r in xr.refs_for(earlier.id, current_user.id)]
+ data['idempotent_replay'] = True
+ return jsonify(data), 200
+
+ if key:
+ replay = _replay()
+ if replay is not None:
+ return replay
+ # The key is unique per user (#476): one older than 24 hours is
+ # cleared so this upload can take it.
+ Recording.query.filter(Recording.user_id == current_user.id, Recording.upload_idempotency_key == key,
+ Recording.created_at < cutoff).update({'upload_idempotency_key': None},
+ synchronize_session=False)
+ db.session.commit()
# Report tags and folders the upload cannot use (the pipeline drops them).
def _usable(item):
@@ -3878,7 +3892,7 @@ def upload_recording():
xr.add(recording, current_user.id, clean)
response = ingest_uploaded_recording(owner=current_user, uploaded_file=request.files.get('file'),
- form=form, prepare=_prepare)
+ form=form, prepare=_prepare, on_conflict=_replay if key else None)
body, status = (response if isinstance(response, tuple) else (response, response.status_code))
if status in (200, 201, 202):
data = body.get_json() or {}
diff --git a/src/api/recordings.py b/src/api/recordings.py
index 7b75dddf..8b3c4528 100644
--- a/src/api/recordings.py
+++ b/src/api/recordings.py
@@ -20,6 +20,7 @@ from flask_login import login_required, current_user
from werkzeug.utils import secure_filename
from werkzeug.exceptions import RequestEntityTooLarge
from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
from email.utils import encode_rfc2231
from urllib.parse import quote
@@ -2224,12 +2225,16 @@ def ingest_uploaded_recording(
success_status=202,
reuse_duplicate=False,
prepare=None,
+ on_conflict=None,
):
"""Run an uploaded file through Speakr's standard ingestion pipeline.
- prepare(recording), when given, runs before the recording is first
- committed, so fields it sets (API v1 participants, references, the
- idempotency key) arrive with the recording itself.
+ prepare(recording), when given, runs before the file is stored and the
+ recording is first committed, so fields it sets (API v1 participants,
+ references, the idempotency key) arrive with the recording itself.
+ on_conflict(), when given, answers an upload whose rows hit a unique
+ index (a concurrent retry with the same idempotency key, #476) with a
+ response, or None for the usual error.
"""
try:
if uploaded_file is None:
@@ -2844,6 +2849,8 @@ def ingest_uploaded_recording(
)
db.session.add(recording)
db.session.flush() # Assign recording.id without committing transaction yet
+ if prepare is not None:
+ prepare(recording)
storage_key = storage.build_recording_key(original_filename, recording.id, now=now)
stored_object = storage.upload_local_file(
@@ -2867,9 +2874,6 @@ def ingest_uploaded_recording(
)
db.session.add(new_association)
- if prepare is not None:
- prepare(recording)
-
db.session.commit()
if selected_tags:
@@ -2932,6 +2936,18 @@ def ingest_uploaded_recording(
'effective_limit_mb': limit_mb,
'audio_only_mode': audio_only_mode,
}), 413
+ except IntegrityError as e:
+ # Another upload committed the same unique key first (#476): keep
+ # nothing of this one and let the caller answer with that recording.
+ db.session.rollback()
+ if 'filepath' in locals() and filepath and os.path.exists(filepath):
+ os.remove(filepath)
+ replay = on_conflict() if on_conflict is not None else None
+ if replay is not None:
+ current_app.logger.info("Upload lost a race on a unique key; answering with the earlier recording")
+ return replay
+ current_app.logger.error(f"Error during file upload: {e}", exc_info=True)
+ return jsonify({'error': 'An unexpected error occurred during upload.'}), 500
except Exception as e:
db.session.rollback()
current_app.logger.error(f"Error during file upload: {e}", exc_info=True)
diff --git a/src/init_db.py b/src/init_db.py
index ef877e01..f943110a 100644
--- a/src/init_db.py
+++ b/src/init_db.py
@@ -26,7 +26,7 @@ from src.utils import (
migration_lock,
run_once,
)
-from src.utils.database import ensure_migration_ledger
+from src.utils.database import drop_index_if_exists, ensure_migration_ledger
# Configuration
ENABLE_INQUIRE_MODE = os.environ.get('ENABLE_INQUIRE_MODE', 'false').lower() == 'true'
@@ -189,6 +189,32 @@ def _migration_section(app, failures, name):
)
+def unique_upload_idempotency_key(app, engine):
+ """One recording per upload idempotency key (#476).
+
+ The first index was not unique, so overlapping retries could store a key
+ twice: older duplicates lose their key (the newest keeps replaying), then
+ the unique index replaces the old one. NULLs are distinct, so unkeyed
+ recordings never clash.
+ """
+ if 'ux_recording_user_idem' not in {i['name'] for i in inspect(engine).get_indexes('recording')}:
+ with engine.begin() as conn:
+ dupes = conn.execute(text(
+ 'SELECT user_id, upload_idempotency_key, MAX(id) FROM recording '
+ 'WHERE upload_idempotency_key IS NOT NULL '
+ 'GROUP BY user_id, upload_idempotency_key HAVING COUNT(*) > 1')).fetchall()
+ for user_id, key, keep in dupes:
+ conn.execute(text(
+ 'UPDATE recording SET upload_idempotency_key = NULL '
+ 'WHERE user_id = :u AND upload_idempotency_key = :k AND id < :keep'),
+ {'u': user_id, 'k': key, 'keep': keep})
+ if dupes:
+ app.logger.info(f"Cleared {len(dupes)} duplicated upload idempotency keys")
+ create_index_if_not_exists(engine, 'ux_recording_user_idem', 'recording',
+ 'user_id, upload_idempotency_key', unique=True)
+ drop_index_if_exists(engine, 'ix_recording_user_idem', 'recording')
+
+
def _run_migrations(app, engine):
"""Apply every schema migration. Assumes the migration lock is held."""
failures = []
@@ -290,7 +316,7 @@ def _run_migrations(app, engine):
# Idempotent API uploads (mailr spec G8).
if add_column_if_not_exists(engine, 'recording', 'upload_idempotency_key', 'VARCHAR(100)'):
app.logger.info("Added upload_idempotency_key column to recording table")
- create_index_if_not_exists(engine, 'ix_recording_user_idem', 'recording', 'user_id, upload_idempotency_key')
+ unique_upload_idempotency_key(app, engine)
# Chunks are read and deleted by recording, in order (D#441)
create_index_if_not_exists(engine, 'ix_transcript_chunk_recording', 'transcript_chunk',
'recording_id, chunk_index')
diff --git a/src/models/recording.py b/src/models/recording.py
index ba3f82b9..37d76bb7 100644
--- a/src/models/recording.py
+++ b/src/models/recording.py
@@ -53,7 +53,8 @@ class Recording(db.Model):
# listener in src/services/recording_changes.py, never by hand.
updated_at = db.Column(db.DateTime, nullable=True, index=True)
# API upload retries (mailr spec G8): a repeat with the same key by the same
- # user within 24 hours returns this recording.
+ # user within 24 hours returns this recording. Unique per user
+ # (ux_recording_user_idem, #476); an expired key is cleared before reuse.
upload_idempotency_key = db.Column(db.String(100), nullable=True)
processing_time_seconds = db.Column(db.Integer, nullable=True)
transcription_duration_seconds = db.Column(db.Integer, nullable=True) # Time taken for transcription
diff --git a/src/utils/database.py b/src/utils/database.py
index d911954c..37e80b92 100644
--- a/src/utils/database.py
+++ b/src/utils/database.py
@@ -322,6 +322,22 @@ def create_index_if_not_exists(engine, index_name, table_name, columns, unique=F
return True
+def drop_index_if_exists(engine, index_name, table_name):
+ """Drop an index if it exists. Returns True if it was dropped."""
+ inspector = inspect(engine)
+ if table_name not in inspector.get_table_names():
+ return False
+ if index_name not in [idx['name'] for idx in inspector.get_indexes(table_name)]:
+ return False
+ with engine.connect() as conn:
+ if engine.name == 'mysql':
+ conn.execute(text(f'DROP INDEX {index_name} ON `{table_name}`'))
+ else:
+ conn.execute(text(f'DROP INDEX {index_name}'))
+ conn.commit()
+ return True
+
+
def drop_not_null(engine, table_name, column_name):
"""
Make an existing NOT NULL column nullable.
diff --git a/tests/test_api_v1_external_refs.py b/tests/test_api_v1_external_refs.py
index 9a8a8fd4..f062bf7e 100644
--- a/tests/test_api_v1_external_refs.py
+++ b/tests/test_api_v1_external_refs.py
@@ -253,6 +253,9 @@ def test_an_old_key_does_not_replay(world):
with _no_media_tools():
second = _upload(world, idempotency_key="msg-old")
assert second.status_code == 202 and second.get_json()["id"] != first.get_json()["id"]
+ with app.app_context():
+ keys = {r.id: r.upload_idempotency_key for r in Recording.query.filter_by(user_id=world["owner"])}
+ assert keys[first.get_json()["id"]] is None and keys[second.get_json()["id"]] == "msg-old"
def test_strict_and_lenient_tags(world):
diff --git a/tests/test_upgrade_mailr_api.py b/tests/test_upgrade_mailr_api.py
index bd81a093..de20c247 100644
--- a/tests/test_upgrade_mailr_api.py
+++ b/tests/test_upgrade_mailr_api.py
@@ -152,7 +152,8 @@ def test_upload_idempotency_and_external_refs(tmp_path, fixture):
_upgrade(db_path)
con = sqlite3.connect(db_path)
assert "upload_idempotency_key" in _columns(con, "recording")
- assert "ix_recording_user_idem" in {r[1] for r in con.execute("PRAGMA index_list('recording')")}
+ indexes = {r[1]: r[2] for r in con.execute("PRAGMA index_list('recording')")}
+ assert indexes.get("ux_recording_user_idem") == 1 and "ix_recording_user_idem" not in indexes
assert {"recording_id", "user_id", "system", "kind", "ref", "url", "label"} <= _columns(con, "recording_external_ref")
con.close()
diff --git a/tests/test_upload_idempotency_race.py b/tests/test_upload_idempotency_race.py
new file mode 100644
index 00000000..6058214b
--- /dev/null
+++ b/tests/test_upload_idempotency_race.py
@@ -0,0 +1,106 @@
+"""One recording per upload idempotency key, also under overlapping retries (#476).
+
+The endpoint checks the key long before the recording is written, so two
+retries can both pass the check. The unique index decides: the loser answers
+with the winner's recording and leaves nothing of its own behind.
+
+SHARED-DB: uses the world fixture of test_api_v1_external_refs, which removes
+its users, recordings, webhooks and stored media afterwards.
+"""
+
+import json
+import os
+import sys
+from unittest.mock import patch
+
+import pytest
+from sqlalchemy import inspect, text
+from sqlalchemy.exc import IntegrityError
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.app import app, db
+from src.init_db import unique_upload_idempotency_key
+from src.models import Recording, Webhook, WebhookDelivery
+from src.services.storage import get_storage_service
+from src.services.storage.service import StorageService
+from tests.test_api_v1_external_refs import _no_media_tools, _upload, world # noqa: F401
+
+CODEC = {"has_video": False, "audio_codec": "opus", "video_codec": None, "duration": 5.0}
+
+
+def test_a_retry_that_overlaps_the_first_upload_replays_it(world):
+ with app.app_context():
+ hook = Webhook(user_id=world["owner"], name="m", url="https://example.com/h", secret="s" * 40,
+ events=json.dumps(["recording.created"]))
+ db.session.add(hook)
+ db.session.commit()
+ hook_id = hook.id
+ staging = get_storage_service().get_staging_dir()
+ staged_before = set(os.listdir(staging))
+ store = StorageService.upload_local_file
+ winner = []
+
+ def _codec(*args, **kwargs):
+ # The first request has passed its key check; the retry runs to the end now.
+ if not winner:
+ winner.append(None)
+ winner[0] = _upload(world, idempotency_key="msg-race")
+ return CODEC
+
+ with _no_media_tools() as enqueue, \
+ patch("src.api.recordings.get_codec_info", side_effect=_codec), \
+ patch.object(StorageService, "upload_local_file", autospec=True, side_effect=store) as stored:
+ loser = _upload(world, idempotency_key="msg-race")
+ assert enqueue.call_count == 1 and stored.call_count == 1
+
+ first = winner[0]
+ assert first.status_code == 202
+ assert loser.status_code == 200 and loser.get_json()["idempotent_replay"] is True
+ assert loser.get_json()["id"] == first.get_json()["id"]
+ assert set(os.listdir(staging)) == staged_before
+ with app.app_context():
+ assert Recording.query.filter_by(user_id=world["owner"], original_filename="memo.webm").count() == 1
+ assert WebhookDelivery.query.filter_by(webhook_id=hook_id).count() == 1
+
+
+def _index(engine, name):
+ return next((i for i in inspect(engine).get_indexes("recording") if i["name"] == name), None)
+
+
+def test_the_migration_keeps_the_newest_of_duplicated_keys(world):
+ with app.app_context():
+ engine = db.engine
+ try:
+ # The schema before #476: a plain index that let a key in twice.
+ with engine.begin() as conn:
+ conn.execute(text("DROP INDEX ux_recording_user_idem"))
+ conn.execute(text('CREATE INDEX ix_recording_user_idem ON "recording" '
+ "(user_id, upload_idempotency_key)"))
+ rows = [Recording(user_id=user, title="r", status="COMPLETED", original_filename="r.wav",
+ upload_idempotency_key=key)
+ for user, key in [("owner", "dup"), ("owner", "dup"), ("owner", "dup"),
+ ("owner", "solo"), ("guest", "dup")]]
+ for r, user in zip(rows, ["owner"] * 4 + ["guest"]):
+ r.user_id = world[user]
+ db.session.add_all(rows)
+ db.session.flush()
+ ids = [r.id for r in rows]
+ db.session.commit() # an open session would block the DDL on PostgreSQL
+
+ unique_upload_idempotency_key(app, engine)
+ unique_upload_idempotency_key(app, engine) # second start: no-op
+
+ db.session.expire_all()
+ keys = [db.session.get(Recording, i).upload_idempotency_key for i in ids]
+ assert keys == [None, None, "dup", "solo", "dup"]
+ assert _index(engine, "ux_recording_user_idem")["unique"]
+ assert _index(engine, "ix_recording_user_idem") is None
+ db.session.add(Recording(user_id=world["owner"], title="r", status="COMPLETED",
+ original_filename="r.wav", upload_idempotency_key="solo"))
+ with pytest.raises(IntegrityError):
+ db.session.commit()
+ db.session.rollback()
+ finally:
+ db.session.rollback()
+ unique_upload_idempotency_key(app, engine)