Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

fix: make the ingest wait survive a failing proxy and a dead worker

François Catuhe4801cc07710c
diff --git a/docs/admin-guide/recording-sessions.md b/docs/admin-guide/recording-sessions.md
index e96fa9d2..2fe3dcf9 100644
--- a/docs/admin-guide/recording-sessions.md
+++ b/docs/admin-guide/recording-sessions.md
@@ -132,6 +132,7 @@ Environment variables, all optional:
| `RECORDING_SESSION_MAX_CHUNK_BYTES` | `16777216` (16 MB) | Per-chunk upload cap. Generous; MediaRecorder chunks are typically <1 MB. |
| `RECORDING_SESSION_ALLOWED_MIME_TYPES` | `audio/webm,audio/ogg,audio/mp4,audio/mpeg,audio/wav,audio/x-m4a,video/webm,video/mp4` | Comma-separated whitelist. The video types carry the optional tab/window/screen video capture. |
| `RECORDING_SESSION_CLEANUP_INTERVAL_SECONDS` | `3600` | How often the background thread sweeps expired sessions. Set to `0` to disable. |
+| `RECORDING_SESSION_INGEST_CEILING_SECONDS` | `900` | How long a sliced upload's in-request ingest can possibly still be running. Past this, a session left `finalizing` is taken over by the next finalize. Keep it above the WSGI request timeout (gunicorn ships with `--timeout 600`), never below, or a live ingest could be claimed twice. |
| `RECORDING_MAX_HOURS` | `8` | Absolute ceiling on a single recording. Stops the recorder automatically at this duration regardless of size; a toast warns the user at 80% of the ceiling. |
| `RECORDING_VIDEO_KBPS` | `2500` | Video bitrate cap (kbps) for the opt-in tab/window/screen video capture. At the default, an hour of capture is roughly 1 GB. |
@@ -246,6 +247,17 @@ on the server, or discard them.
cap, the API returns 507 on `POST /upload/session` and on chunk
uploads that would exceed the cap. Surfaced to the user as a quota
banner.
+- **Deploys interrupt sliced uploads that are ingesting.** A sliced
+ upload ingests inside the finalize request, so recreating the
+ container mid-ingest loses that work and leaves the session
+ `finalizing` with nobody running it. Nothing is lost and nothing
+ leaks: the slices stay on disk and the user's retry takes the session
+ over once its claim is older than
+ `RECORDING_SESSION_INGEST_CEILING_SECONDS`, re-ingesting from the
+ slices already delivered. But a retry *before* that point waits ten
+ minutes and then fails, so a deploy is best timed when nothing is
+ uploading. Slice POSTs and recorder sessions are unaffected: those
+ requests are short, and their retries resume.
## API
diff --git a/src/api/recording_sessions.py b/src/api/recording_sessions.py
index 7d744007..bcd94315 100644
--- a/src/api/recording_sessions.py
+++ b/src/api/recording_sessions.py
@@ -107,6 +107,22 @@ def _max_chunk_bytes():
return int(os.environ.get('RECORDING_SESSION_MAX_CHUNK_BYTES', str(16 * 1024 * 1024)))
+def _ingest_ceiling_seconds():
+ """How long a finalize's ingest can possibly still be running.
+
+ A sliced upload's ingest happens inside the request, so no ingest
+ outlives the WSGI server's own request timeout (gunicorn runs with
+ ``--timeout 600``). Past this, a session still marked ``finalizing``
+ belongs to a worker that no longer exists, typically because the
+ container was recreated mid-ingest, and the next finalize may take it
+ over. Set it above the deployment's request timeout, never below.
+ """
+ try:
+ return max(60, int(os.environ.get('RECORDING_SESSION_INGEST_CEILING_SECONDS', '900')))
+ except (TypeError, ValueError):
+ return 900
+
+
def _commit_batch_size():
"""How many chunks to accumulate before committing session bookkeeping.
@@ -611,6 +627,12 @@ def finalize_sliced_upload(session_id):
user's. What the slices weigh together is checked against the size
declared at create, so a short or missing slice fails loudly instead
of ingesting as a truncated recording.
+
+ A session already ``finalizing`` is claimable again once its claim is
+ older than :func:`_ingest_ceiling_seconds`, which no live ingest can
+ be: that state otherwise outlives the worker that owned it whenever a
+ container is recreated mid-ingest, and the user's retry would wait
+ for an ingest nobody is running until the TTL sweep, a day later.
"""
session = db.session.get(RecordingSession, session_id)
err = _ensure_owned(session)
@@ -625,19 +647,26 @@ def finalize_sliced_upload(session_id):
if session.status == 'finalized':
return _replay_sliced_upload_response(session)
- if session.status != 'recording':
+ if session.status not in ('recording', 'finalizing'):
return jsonify({
'error': f'Session is in status {session.status!r}; cannot finalize',
}), 409
+ now = datetime.utcnow()
+ abandoned_before = now - timedelta(seconds=_ingest_ceiling_seconds())
claimed = db.session.query(RecordingSession).filter(
RecordingSession.id == session.id,
- RecordingSession.status == 'recording',
- ).update({'status': 'finalizing'}, synchronize_session=False)
+ db.or_(
+ RecordingSession.status == 'recording',
+ db.and_(
+ RecordingSession.status == 'finalizing',
+ RecordingSession.last_seen_at < abandoned_before,
+ ),
+ ),
+ ).update({'status': 'finalizing', 'last_seen_at': now}, synchronize_session=False)
if not claimed:
db.session.rollback()
return jsonify({'error': 'Session is already being finalized'}), 409
- session.last_seen_at = datetime.utcnow()
db.session.commit()
dir_path = _session_dir(session.id)
diff --git a/static/js/modules/db/sliced-file-upload.js b/static/js/modules/db/sliced-file-upload.js
index 01eaf970..29f107de 100644
--- a/static/js/modules/db/sliced-file-upload.js
+++ b/static/js/modules/db/sliced-file-upload.js
@@ -108,20 +108,34 @@ export function forgetSession(file) {
writeResumeMemory(memory);
}
-/** The server's view of a session, or null when it no longer has one. */
+/**
+ * The server's view of a session.
+ *
+ * Null means the server says there is no such session, which is a fact
+ * about the session. Every other failure throws, because a status
+ * endpoint that cannot be reached says nothing about the upload behind
+ * it, and the difference decides whether a caller gives up or waits.
+ */
async function readSessionStatus(sessionId, token) {
const response = await fetch(`${SESSION_BASE}/${encodeURIComponent(sessionId)}`, {
credentials: 'same-origin',
headers: { 'X-CSRFToken': token },
});
- if (!response.ok) return null;
- return parseJson(await response.text());
+ if (response.status === 404 || response.status === 403) return null;
+ if (!response.ok) {
+ const error = new Error(`Could not read the upload session (HTTP ${response.status})`);
+ error.status = response.status;
+ throw error;
+ }
+ const session = parseJson(await response.text());
+ if (!session) throw new Error('Upload session status was not JSON');
+ return session;
}
/**
* The session we opened for this file, when it can still be continued.
- * Returns `{sessionId, sliceBytes, chunkCount, status}` or null, and
- * forgets anything the server no longer recognises.
+ * Returns `{sessionId, sliceBytes, chunkCount}` or null, and forgets
+ * anything the server no longer recognises.
*
* `finalizing` counts as continuable: it is what an ingest that outran
* the proxy's patience looks like from here, and the caller waits it out
@@ -153,7 +167,6 @@ async function resumableSession(file, token) {
sessionId: remembered.sessionId,
sliceBytes: remembered.sliceBytes,
chunkCount: status.chunk_count || 0,
- status: status.status,
};
}
@@ -356,6 +369,11 @@ async function finalize(sessionId, formData, tokenRef, options) {
* finalizes for real (`recording`, meaning the request never reached
* Speakr at all). Only when the wait runs out does `firstError` surface,
* with the session left in place for another attempt.
+ *
+ * A status endpoint that answers with anything but "no such session" is
+ * treated as part of the same outage: this code runs precisely where a
+ * proxy is having a bad time, so ending the wait on the first 502 from
+ * it would abandon an ingest that is still running.
*/
async function settleUnfinishedFinalize(sessionId, formData, tokenRef, options, firstError) {
const deadline = Date.now() + INGEST_WAIT_MS;
@@ -366,22 +384,25 @@ async function settleUnfinishedFinalize(sessionId, formData, tokenRef, options,
try {
status = await readSessionStatus(sessionId, tokenRef.token);
} catch (_) {
- status = undefined;
+ await sleep(INGEST_POLL_MS);
+ continue;
}
if (status === null) throw firstError;
-
- if (status?.status === 'finalizing' || status === undefined) {
+ if (status.status === 'finalizing') {
await sleep(INGEST_POLL_MS);
continue;
}
+ if (!['recording', 'finalized'].includes(status.status)) throw firstError;
+ if (finalizeAttempts >= MAX_FINALIZE_ATTEMPTS) throw firstError;
- if (['recording', 'finalized'].includes(status.status)
- && ++finalizeAttempts <= MAX_FINALIZE_ATTEMPTS) {
+ finalizeAttempts += 1;
+ try {
return await finalize(sessionId, formData, tokenRef, options);
+ } catch (error) {
+ if (!error.resumable) throw error;
+ await sleep(INGEST_POLL_MS);
}
-
- throw firstError;
}
throw firstError;
diff --git a/static/js/modules/db/sliced-file-upload.test.js b/static/js/modules/db/sliced-file-upload.test.js
index 4fa43da5..55f7ddc5 100644
--- a/static/js/modules/db/sliced-file-upload.test.js
+++ b/static/js/modules/db/sliced-file-upload.test.js
@@ -61,11 +61,14 @@ function _installMocks() {
};
}
if (!options.method) {
- const status = nextOf(sessionStatus);
+ const answer = nextOf(sessionStatus);
+ if (answer?.httpStatus) {
+ return { status: answer.httpStatus, ok: false, text: async () => 'proxy error' };
+ }
return {
- status: status ? 200 : 404,
- ok: !!status,
- text: async () => JSON.stringify(status || {}),
+ status: answer ? 200 : 404,
+ ok: !!answer,
+ text: async () => JSON.stringify(answer || {}),
};
}
return { status: 204, text: async () => '' };
@@ -344,6 +347,53 @@ describe('resuming an interrupted upload', () => {
expect(rememberedSessions()).toEqual({});
});
+ // The wait runs while a proxy is misbehaving, so its errors are part of that outage, not an answer.
+ it('keeps waiting when the status endpoint itself is failing', async () => {
+ vi.useFakeTimers();
+ finalizeResponse = [TIMED_OUT, INGESTED];
+ sessionStatus = [
+ { httpStatus: 502 },
+ { httpStatus: 504 },
+ { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB },
+ { kind: 'sliced_upload', status: 'finalized', chunk_count: 3, upload_total_bytes: 40 * MB },
+ ];
+
+ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});
+ await vi.advanceTimersByTimeAsync(20000);
+
+ expect(await upload).toMatchObject({ id: 42 });
+ expect(deleteCalls()).toHaveLength(0);
+ });
+
+ it('gives up on the wait when the server says the session is gone', async () => {
+ vi.useFakeTimers();
+ finalizeResponse = TIMED_OUT;
+ sessionStatus = null;
+
+ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});
+ const assertion = expect(upload).rejects.toThrow('HTTP 524');
+ await vi.advanceTimersByTimeAsync(10000);
+ await assertion;
+
+ expect(finalizePosts()).toHaveLength(1);
+ });
+
+ it('keeps polling when a second finalize is cut off as well', async () => {
+ vi.useFakeTimers();
+ finalizeResponse = [TIMED_OUT, TIMED_OUT, INGESTED];
+ sessionStatus = [
+ { kind: 'sliced_upload', status: 'recording', chunk_count: 3, upload_total_bytes: 40 * MB },
+ { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB },
+ { kind: 'sliced_upload', status: 'finalized', chunk_count: 3, upload_total_bytes: 40 * MB },
+ ];
+
+ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});
+ await vi.advanceTimersByTimeAsync(30000);
+
+ expect(await upload).toMatchObject({ id: 42 });
+ expect(finalizePosts()).toHaveLength(3);
+ });
+
it('finalizes for real when the timed-out request never reached Speakr', async () => {
vi.useFakeTimers();
finalizeResponse = [{ status: 502, text: 'Bad Gateway' },
diff --git a/tests/test_sliced_file_upload.py b/tests/test_sliced_file_upload.py
index f5ff0d1f..bb1fc664 100644
--- a/tests/test_sliced_file_upload.py
+++ b/tests/test_sliced_file_upload.py
@@ -402,6 +402,48 @@ def test_an_interrupted_sliced_upload_resumes_from_the_slices_already_sent():
shutil.rmtree(upload_folder, ignore_errors=True)
+def test_a_finalize_abandoned_by_a_dead_worker_can_be_taken_over():
+ """Recreating the container mid-ingest leaves the session finalizing
+ with nobody ingesting. Without a takeover the user's retry waits for
+ a worker that no longer exists until the TTL sweep, a day later."""
+ upload_folder = _tmp_upload_folder()
+ with app.app_context():
+ app.config["UPLOAD_FOLDER"] = upload_folder
+ user = _mk_user("slice_takeover")
+ client = app.test_client()
+ _login(client, user)
+
+ payload = _payload(SLICE_BYTES)
+ session_id = _open_session(client, filename="sample.mp3", mime_type="audio/mpeg",
+ total_bytes=len(payload))["session_id"]
+ _send_slices(client, session_id, payload)
+
+ session = db.session.get(RecordingSession, session_id)
+ session.status = "finalizing"
+ session.last_seen_at = datetime.utcnow()
+ db.session.commit()
+
+ fresh_claim = client.post(f"/upload/session/{session_id}/finalize-upload",
+ data={}, content_type="multipart/form-data")
+ assert fresh_claim.status_code == 409
+ assert "already being finalized" in fresh_claim.get_json()["error"]
+
+ session.last_seen_at = datetime.utcnow() - timedelta(hours=1)
+ db.session.commit()
+
+ staging = os.path.join(upload_folder, f"stg_{uuid.uuid4().hex[:6]}")
+ with _upload_mocks(staging):
+ taken_over = client.post(f"/upload/session/{session_id}/finalize-upload",
+ data={}, content_type="multipart/form-data")
+
+ assert taken_over.status_code == 202, taken_over.data
+ recording = db.session.get(Recording, taken_over.get_json()["id"])
+ assert recording.file_hash == hashlib.sha256(payload).hexdigest()
+
+ _cleanup(recording, user)
+ shutil.rmtree(upload_folder, ignore_errors=True)
+
+
def test_another_user_cannot_finalize_someone_elses_sliced_upload():
upload_folder = _tmp_upload_folder()
with app.app_context():