Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
fix: make the ingest wait survive a failing proxy and a dead worker
François Catuhe4801cc07710c
diff --git a/docs/admin-guide/recording-sessions.md b/docs/admin-guide/recording-sessions.mdindex e96fa9d2..2fe3dcf9 100644--- a/docs/admin-guide/recording-sessions.md+++ b/docs/admin-guide/recording-sessions.md@@ -132,6 +132,7 @@ Environment variables, all optional:| `RECORDING_SESSION_MAX_CHUNK_BYTES` | `16777216` (16 MB) | Per-chunk upload cap. Generous; MediaRecorder chunks are typically <1 MB. || `RECORDING_SESSION_ALLOWED_MIME_TYPES` | `audio/webm,audio/ogg,audio/mp4,audio/mpeg,audio/wav,audio/x-m4a,video/webm,video/mp4` | Comma-separated whitelist. The video types carry the optional tab/window/screen video capture. || `RECORDING_SESSION_CLEANUP_INTERVAL_SECONDS` | `3600` | How often the background thread sweeps expired sessions. Set to `0` to disable. |+| `RECORDING_SESSION_INGEST_CEILING_SECONDS` | `900` | How long a sliced upload's in-request ingest can possibly still be running. Past this, a session left `finalizing` is taken over by the next finalize. Keep it above the WSGI request timeout (gunicorn ships with `--timeout 600`), never below, or a live ingest could be claimed twice. || `RECORDING_MAX_HOURS` | `8` | Absolute ceiling on a single recording. Stops the recorder automatically at this duration regardless of size; a toast warns the user at 80% of the ceiling. || `RECORDING_VIDEO_KBPS` | `2500` | Video bitrate cap (kbps) for the opt-in tab/window/screen video capture. At the default, an hour of capture is roughly 1 GB. |@@ -246,6 +247,17 @@ on the server, or discard them.cap, the API returns 507 on `POST /upload/session` and on chunkuploads that would exceed the cap. Surfaced to the user as a quotabanner.+- **Deploys interrupt sliced uploads that are ingesting.** A sliced+ upload ingests inside the finalize request, so recreating the+ container mid-ingest loses that work and leaves the session+ `finalizing` with nobody running it. Nothing is lost and nothing+ leaks: the slices stay on disk and the user's retry takes the session+ over once its claim is older than+ `RECORDING_SESSION_INGEST_CEILING_SECONDS`, re-ingesting from the+ slices already delivered. But a retry *before* that point waits ten+ minutes and then fails, so a deploy is best timed when nothing is+ uploading. Slice POSTs and recorder sessions are unaffected: those+ requests are short, and their retries resume.## APIdiff --git a/src/api/recording_sessions.py b/src/api/recording_sessions.pyindex 7d744007..bcd94315 100644--- a/src/api/recording_sessions.py+++ b/src/api/recording_sessions.py@@ -107,6 +107,22 @@ def _max_chunk_bytes():return int(os.environ.get('RECORDING_SESSION_MAX_CHUNK_BYTES', str(16 * 1024 * 1024)))+def _ingest_ceiling_seconds():+ """How long a finalize's ingest can possibly still be running.++ A sliced upload's ingest happens inside the request, so no ingest+ outlives the WSGI server's own request timeout (gunicorn runs with+ ``--timeout 600``). Past this, a session still marked ``finalizing``+ belongs to a worker that no longer exists, typically because the+ container was recreated mid-ingest, and the next finalize may take it+ over. Set it above the deployment's request timeout, never below.+ """+ try:+ return max(60, int(os.environ.get('RECORDING_SESSION_INGEST_CEILING_SECONDS', '900')))+ except (TypeError, ValueError):+ return 900++def _commit_batch_size():"""How many chunks to accumulate before committing session bookkeeping.@@ -611,6 +627,12 @@ def finalize_sliced_upload(session_id):user's. What the slices weigh together is checked against the sizedeclared at create, so a short or missing slice fails loudly insteadof ingesting as a truncated recording.++ A session already ``finalizing`` is claimable again once its claim is+ older than :func:`_ingest_ceiling_seconds`, which no live ingest can+ be: that state otherwise outlives the worker that owned it whenever a+ container is recreated mid-ingest, and the user's retry would wait+ for an ingest nobody is running until the TTL sweep, a day later."""session = db.session.get(RecordingSession, session_id)err = _ensure_owned(session)@@ -625,19 +647,26 @@ def finalize_sliced_upload(session_id):if session.status == 'finalized':return _replay_sliced_upload_response(session)- if session.status != 'recording':+ if session.status not in ('recording', 'finalizing'):return jsonify({'error': f'Session is in status {session.status!r}; cannot finalize',}), 409+ now = datetime.utcnow()+ abandoned_before = now - timedelta(seconds=_ingest_ceiling_seconds())claimed = db.session.query(RecordingSession).filter(RecordingSession.id == session.id,- RecordingSession.status == 'recording',- ).update({'status': 'finalizing'}, synchronize_session=False)+ db.or_(+ RecordingSession.status == 'recording',+ db.and_(+ RecordingSession.status == 'finalizing',+ RecordingSession.last_seen_at < abandoned_before,+ ),+ ),+ ).update({'status': 'finalizing', 'last_seen_at': now}, synchronize_session=False)if not claimed:db.session.rollback()return jsonify({'error': 'Session is already being finalized'}), 409- session.last_seen_at = datetime.utcnow()db.session.commit()dir_path = _session_dir(session.id)diff --git a/static/js/modules/db/sliced-file-upload.js b/static/js/modules/db/sliced-file-upload.jsindex 01eaf970..29f107de 100644--- a/static/js/modules/db/sliced-file-upload.js+++ b/static/js/modules/db/sliced-file-upload.js@@ -108,20 +108,34 @@ export function forgetSession(file) {writeResumeMemory(memory);}-/** The server's view of a session, or null when it no longer has one. */+/**+ * The server's view of a session.+ *+ * Null means the server says there is no such session, which is a fact+ * about the session. Every other failure throws, because a status+ * endpoint that cannot be reached says nothing about the upload behind+ * it, and the difference decides whether a caller gives up or waits.+ */async function readSessionStatus(sessionId, token) {const response = await fetch(`${SESSION_BASE}/${encodeURIComponent(sessionId)}`, {credentials: 'same-origin',headers: { 'X-CSRFToken': token },});- if (!response.ok) return null;- return parseJson(await response.text());+ if (response.status === 404 || response.status === 403) return null;+ if (!response.ok) {+ const error = new Error(`Could not read the upload session (HTTP ${response.status})`);+ error.status = response.status;+ throw error;+ }+ const session = parseJson(await response.text());+ if (!session) throw new Error('Upload session status was not JSON');+ return session;}/*** The session we opened for this file, when it can still be continued.- * Returns `{sessionId, sliceBytes, chunkCount, status}` or null, and- * forgets anything the server no longer recognises.+ * Returns `{sessionId, sliceBytes, chunkCount}` or null, and forgets+ * anything the server no longer recognises.** `finalizing` counts as continuable: it is what an ingest that outran* the proxy's patience looks like from here, and the caller waits it out@@ -153,7 +167,6 @@ async function resumableSession(file, token) {sessionId: remembered.sessionId,sliceBytes: remembered.sliceBytes,chunkCount: status.chunk_count || 0,- status: status.status,};}@@ -356,6 +369,11 @@ async function finalize(sessionId, formData, tokenRef, options) {* finalizes for real (`recording`, meaning the request never reached* Speakr at all). Only when the wait runs out does `firstError` surface,* with the session left in place for another attempt.+ *+ * A status endpoint that answers with anything but "no such session" is+ * treated as part of the same outage: this code runs precisely where a+ * proxy is having a bad time, so ending the wait on the first 502 from+ * it would abandon an ingest that is still running.*/async function settleUnfinishedFinalize(sessionId, formData, tokenRef, options, firstError) {const deadline = Date.now() + INGEST_WAIT_MS;@@ -366,22 +384,25 @@ async function settleUnfinishedFinalize(sessionId, formData, tokenRef, options,try {status = await readSessionStatus(sessionId, tokenRef.token);} catch (_) {- status = undefined;+ await sleep(INGEST_POLL_MS);+ continue;}if (status === null) throw firstError;-- if (status?.status === 'finalizing' || status === undefined) {+ if (status.status === 'finalizing') {await sleep(INGEST_POLL_MS);continue;}+ if (!['recording', 'finalized'].includes(status.status)) throw firstError;+ if (finalizeAttempts >= MAX_FINALIZE_ATTEMPTS) throw firstError;- if (['recording', 'finalized'].includes(status.status)- && ++finalizeAttempts <= MAX_FINALIZE_ATTEMPTS) {+ finalizeAttempts += 1;+ try {return await finalize(sessionId, formData, tokenRef, options);+ } catch (error) {+ if (!error.resumable) throw error;+ await sleep(INGEST_POLL_MS);}-- throw firstError;}throw firstError;diff --git a/static/js/modules/db/sliced-file-upload.test.js b/static/js/modules/db/sliced-file-upload.test.jsindex 4fa43da5..55f7ddc5 100644--- a/static/js/modules/db/sliced-file-upload.test.js+++ b/static/js/modules/db/sliced-file-upload.test.js@@ -61,11 +61,14 @@ function _installMocks() {};}if (!options.method) {- const status = nextOf(sessionStatus);+ const answer = nextOf(sessionStatus);+ if (answer?.httpStatus) {+ return { status: answer.httpStatus, ok: false, text: async () => 'proxy error' };+ }return {- status: status ? 200 : 404,- ok: !!status,- text: async () => JSON.stringify(status || {}),+ status: answer ? 200 : 404,+ ok: !!answer,+ text: async () => JSON.stringify(answer || {}),};}return { status: 204, text: async () => '' };@@ -344,6 +347,53 @@ describe('resuming an interrupted upload', () => {expect(rememberedSessions()).toEqual({});});+ // The wait runs while a proxy is misbehaving, so its errors are part of that outage, not an answer.+ it('keeps waiting when the status endpoint itself is failing', async () => {+ vi.useFakeTimers();+ finalizeResponse = [TIMED_OUT, INGESTED];+ sessionStatus = [+ { httpStatus: 502 },+ { httpStatus: 504 },+ { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB },+ { kind: 'sliced_upload', status: 'finalized', chunk_count: 3, upload_total_bytes: 40 * MB },+ ];++ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});+ await vi.advanceTimersByTimeAsync(20000);++ expect(await upload).toMatchObject({ id: 42 });+ expect(deleteCalls()).toHaveLength(0);+ });++ it('gives up on the wait when the server says the session is gone', async () => {+ vi.useFakeTimers();+ finalizeResponse = TIMED_OUT;+ sessionStatus = null;++ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});+ const assertion = expect(upload).rejects.toThrow('HTTP 524');+ await vi.advanceTimersByTimeAsync(10000);+ await assertion;++ expect(finalizePosts()).toHaveLength(1);+ });++ it('keeps polling when a second finalize is cut off as well', async () => {+ vi.useFakeTimers();+ finalizeResponse = [TIMED_OUT, TIMED_OUT, INGESTED];+ sessionStatus = [+ { kind: 'sliced_upload', status: 'recording', chunk_count: 3, upload_total_bytes: 40 * MB },+ { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB },+ { kind: 'sliced_upload', status: 'finalized', chunk_count: 3, upload_total_bytes: 40 * MB },+ ];++ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});+ await vi.advanceTimersByTimeAsync(30000);++ expect(await upload).toMatchObject({ id: 42 });+ expect(finalizePosts()).toHaveLength(3);+ });+it('finalizes for real when the timed-out request never reached Speakr', async () => {vi.useFakeTimers();finalizeResponse = [{ status: 502, text: 'Bad Gateway' },diff --git a/tests/test_sliced_file_upload.py b/tests/test_sliced_file_upload.pyindex f5ff0d1f..bb1fc664 100644--- a/tests/test_sliced_file_upload.py+++ b/tests/test_sliced_file_upload.py@@ -402,6 +402,48 @@ def test_an_interrupted_sliced_upload_resumes_from_the_slices_already_sent():shutil.rmtree(upload_folder, ignore_errors=True)+def test_a_finalize_abandoned_by_a_dead_worker_can_be_taken_over():+ """Recreating the container mid-ingest leaves the session finalizing+ with nobody ingesting. Without a takeover the user's retry waits for+ a worker that no longer exists until the TTL sweep, a day later."""+ upload_folder = _tmp_upload_folder()+ with app.app_context():+ app.config["UPLOAD_FOLDER"] = upload_folder+ user = _mk_user("slice_takeover")+ client = app.test_client()+ _login(client, user)++ payload = _payload(SLICE_BYTES)+ session_id = _open_session(client, filename="sample.mp3", mime_type="audio/mpeg",+ total_bytes=len(payload))["session_id"]+ _send_slices(client, session_id, payload)++ session = db.session.get(RecordingSession, session_id)+ session.status = "finalizing"+ session.last_seen_at = datetime.utcnow()+ db.session.commit()++ fresh_claim = client.post(f"/upload/session/{session_id}/finalize-upload",+ data={}, content_type="multipart/form-data")+ assert fresh_claim.status_code == 409+ assert "already being finalized" in fresh_claim.get_json()["error"]++ session.last_seen_at = datetime.utcnow() - timedelta(hours=1)+ db.session.commit()++ staging = os.path.join(upload_folder, f"stg_{uuid.uuid4().hex[:6]}")+ with _upload_mocks(staging):+ taken_over = client.post(f"/upload/session/{session_id}/finalize-upload",+ data={}, content_type="multipart/form-data")++ assert taken_over.status_code == 202, taken_over.data+ recording = db.session.get(Recording, taken_over.get_json()["id"])+ assert recording.file_hash == hashlib.sha256(payload).hexdigest()++ _cleanup(recording, user)+ shutil.rmtree(upload_folder, ignore_errors=True)++def test_another_user_cannot_finalize_someone_elses_sliced_upload():upload_folder = _tmp_upload_folder()with app.app_context():