Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
fix: wait out an ingest that outran the proxy instead of reporting failure
François Catuhe459169ea7bbd
diff --git a/docs/admin-guide/recording-sessions.md b/docs/admin-guide/recording-sessions.mdindex 67a28e44..e96fa9d2 100644--- a/docs/admin-guide/recording-sessions.md+++ b/docs/admin-guide/recording-sessions.md@@ -92,12 +92,19 @@ ingesting at that moment, and the retry gets that recording back.the origin finishes ingesting anyway.Speakr treats 524 and the rest of Cloudflare's 52x range as- inconclusive rather than fatal, so the session survives and the- user's retry replays the recording that landed instead of sending- the file again. Time your worst realistic upload before putting the- edge in front of it: if it breaches the budget, the fix is deferring- conversion to the job queue, which `VIDEO_RETENTION=true` already- does for videos whose original you want to keep.+ inconclusive rather than fatal. The browser polls the session until+ the ingest settles, for up to ten minutes, and then either takes the+ recording that landed or finalizes for real if the request never+ arrived. Nothing is re-uploaded and the user is not asked to retry;+ the upload simply takes as long as the ingest does. Re-adding the+ same file while an ingest is still running waits on it too, rather+ than starting a second copy.++ Time your worst realistic upload before putting the edge in front of+ it anyway: a wait that outlives ten minutes still surfaces as a+ failure, and the fix for that is deferring conversion to the job+ queue, which `VIDEO_RETENTION=true` already does for videos whose+ original you want to keep.Operationally this means in-progress sliced uploads sit in`UPLOAD_FOLDER/_sessions/` for up to `RECORDING_SESSION_TTL_HOURS`diff --git a/static/js/modules/db/sliced-file-upload.js b/static/js/modules/db/sliced-file-upload.jsindex 8e60f0f2..01eaf970 100644--- a/static/js/modules/db/sliced-file-upload.js+++ b/static/js/modules/db/sliced-file-upload.js@@ -47,6 +47,10 @@ const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504, 520, 521,const INCONCLUSIVE_STATUSES = new Set([408, 429, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526, 527]);+const INGEST_POLL_MS = 3000;+const INGEST_WAIT_MS = 10 * 60 * 1000;+const MAX_FINALIZE_ATTEMPTS = 3;+const RESUME_KEY = 'speakr.slicedUploads';const RESUME_TTL_MS = 24 * 60 * 60 * 1000;const RESUME_MAX_ENTRIES = 20;@@ -104,10 +108,24 @@ export function forgetSession(file) {writeResumeMemory(memory);}+/** The server's view of a session, or null when it no longer has one. */+async function readSessionStatus(sessionId, token) {+ const response = await fetch(`${SESSION_BASE}/${encodeURIComponent(sessionId)}`, {+ credentials: 'same-origin',+ headers: { 'X-CSRFToken': token },+ });+ if (!response.ok) return null;+ return parseJson(await response.text());+}+/**- * The server's view of a session we started for this file, when it can- * still be continued. Returns `{sessionId, sliceBytes, chunkCount}` or- * null, and forgets anything the server no longer recognises.+ * The session we opened for this file, when it can still be continued.+ * Returns `{sessionId, sliceBytes, chunkCount, status}` or null, and+ * forgets anything the server no longer recognises.+ *+ * `finalizing` counts as continuable: it is what an ingest that outran+ * the proxy's patience looks like from here, and the caller waits it out+ * rather than sending the file again.*/async function resumableSession(file, token) {const remembered = readResumeMemory()[fileKey(file)];@@ -115,14 +133,7 @@ async function resumableSession(file, token) {let status;try {- const response = await fetch(- `${SESSION_BASE}/${encodeURIComponent(remembered.sessionId)}`,- { credentials: 'same-origin', headers: { 'X-CSRFToken': token } });- if (!response.ok) {- forgetSession(file);- return null;- }- status = parseJson(await response.text());+ status = await readSessionStatus(remembered.sessionId, token);} catch (error) {console.warn('[SlicedUpload] Could not read the remembered session:', error);return null;@@ -131,7 +142,7 @@ async function resumableSession(file, token) {const continuable = status&& status.kind === 'sliced_upload'&& status.upload_total_bytes === file.size- && ['recording', 'finalized'].includes(status.status)+ && ['recording', 'finalizing', 'finalized'].includes(status.status)&& remembered.sliceBytes > 0;if (!continuable) {forgetSession(file);@@ -142,6 +153,7 @@ async function resumableSession(file, token) {sessionId: remembered.sessionId,sliceBytes: remembered.sliceBytes,chunkCount: status.chunk_count || 0,+ status: status.status,};}@@ -333,6 +345,48 @@ async function finalize(sessionId, formData, tokenRef, options) {}}+/**+ * Settle a finalize whose answer never arrived.+ *+ * The ingest runs in-request, so Cloudflare's 125-second Proxy Read+ * Timeout can cut the answer off while the origin carries on and+ * produces the recording anyway. Rather than report a failure the user+ * has to interpret, wait the session out: once it leaves `finalizing`,+ * asking again either replays the recording that landed (`finalized`) or+ * finalizes for real (`recording`, meaning the request never reached+ * Speakr at all). Only when the wait runs out does `firstError` surface,+ * with the session left in place for another attempt.+ */+async function settleUnfinishedFinalize(sessionId, formData, tokenRef, options, firstError) {+ const deadline = Date.now() + INGEST_WAIT_MS;+ let finalizeAttempts = 1;++ while (Date.now() < deadline) {+ let status;+ try {+ status = await readSessionStatus(sessionId, tokenRef.token);+ } catch (_) {+ status = undefined;+ }++ if (status === null) throw firstError;++ if (status?.status === 'finalizing' || status === undefined) {+ await sleep(INGEST_POLL_MS);+ continue;+ }++ if (['recording', 'finalized'].includes(status.status)+ && ++finalizeAttempts <= MAX_FINALIZE_ATTEMPTS) {+ return await finalize(sessionId, formData, tokenRef, options);+ }++ throw firstError;+ }++ throw firstError;+}+/*** Upload `file` in slices and finalize with `formData` (the same form the* single-shot path builds, minus the file part). Resolves to the created@@ -381,10 +435,18 @@ export async function uploadFileInSlices(file, formData, options = {}) {}onProgress(Math.min(1, ((index - 1) * sliceBytes) / file.size));}- const recording = await finalize(sessionId, formData, tokenRef, {+ const finalizeOptions = {finalizeTimeoutMs: computeUploadTimeout(file.size),onXhr: options.onXhr,- });+ };+ let recording;+ try {+ recording = await finalize(sessionId, formData, tokenRef, finalizeOptions);+ } catch (error) {+ if (!error.resumable) throw error;+ recording = await settleUnfinishedFinalize(+ sessionId, formData, tokenRef, finalizeOptions, error);+ }forgetSession(file);return recording;} catch (error) {diff --git a/static/js/modules/db/sliced-file-upload.test.js b/static/js/modules/db/sliced-file-upload.test.jsindex 0f29b8d9..4fa43da5 100644--- a/static/js/modules/db/sliced-file-upload.test.js+++ b/static/js/modules/db/sliced-file-upload.test.js@@ -61,10 +61,11 @@ function _installMocks() {};}if (!options.method) {+ const status = nextOf(sessionStatus);return {- status: sessionStatus ? 200 : 404,- ok: !!sessionStatus,- text: async () => JSON.stringify(sessionStatus || {}),+ status: status ? 200 : 404,+ ok: !!status,+ text: async () => JSON.stringify(status || {}),};}return { status: 204, text: async () => '' };@@ -85,7 +86,7 @@ function _installMocks() {const call = { url: this.url, body, headers: this.headers };sent.push(call);const response = this.url.endsWith('/finalize-upload')- ? finalizeResponse+ ? nextOf(finalizeResponse): (sliceResponses.shift() || defaultSliceResponse);queueMicrotask(() => {if (response.networkError) {@@ -103,7 +104,14 @@ function _installMocks() {};}+/** Queued answers are consumed in order; the last one repeats. */+function nextOf(answers) {+ if (!Array.isArray(answers)) return answers;+ return answers.length > 1 ? answers.shift() : answers[0];+}+const slicePosts = () => sent.filter((call) => !call.url.endsWith('/finalize-upload'));+const finalizePosts = () => sent.filter((call) => call.url.endsWith('/finalize-upload'));const createCalls = () => fetchCalls.filter((call) => call.options.method === 'POST');const deleteCalls = () => fetchCalls.filter((call) => call.options.method === 'DELETE');const rememberedSessions = () => JSON.parse(store.get('speakr.slicedUploads') || '{}');@@ -314,26 +322,80 @@ describe('resuming an interrupted upload', () => {});});- // Cloudflare answers 524 at 125s while the origin is still ingesting, and the finalize it timed out on may well have landed.- it('keeps the session when the edge times the finalize out', async () => {- finalizeResponse = { status: 524, text: '<html><title>524: A timeout occurred</title></html>' };+ const TIMED_OUT = { status: 524, text: '<html><title>524: A timeout occurred</title></html>' };+ const INGESTED = { status: 202, text: JSON.stringify({ id: 42, title: 'talk', idempotent_replay: true }) };++ // Cloudflare cuts the answer off at 125s while the origin finishes the ingest, so the recording exists and the client must not re-send the file.+ it('waits out an ingest the edge timed out on and returns its recording', async () => {+ vi.useFakeTimers();+ finalizeResponse = [TIMED_OUT, INGESTED];+ sessionStatus = [+ { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB },+ { kind: 'sliced_upload', status: 'finalized', chunk_count: 3, upload_total_bytes: 40 * MB },+ ];++ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});+ await vi.advanceTimersByTimeAsync(10000);++ expect(await upload).toMatchObject({ id: 42, idempotent_replay: true });+ expect(slicePosts()).toHaveLength(3);+ expect(finalizePosts()).toHaveLength(2);+ expect(deleteCalls()).toHaveLength(0);+ expect(rememberedSessions()).toEqual({});+ });++ it('finalizes for real when the timed-out request never reached Speakr', async () => {+ vi.useFakeTimers();+ finalizeResponse = [{ status: 502, text: 'Bad Gateway' },+ { status: 202, text: JSON.stringify({ id: 43 }) }];+ sessionStatus = { kind: 'sliced_upload', status: 'recording', chunk_count: 3, upload_total_bytes: 40 * MB };++ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});+ await vi.advanceTimersByTimeAsync(10000);++ expect(await upload).toMatchObject({ id: 43 });+ expect(slicePosts()).toHaveLength(3);+ });++ it('surfaces the timeout and keeps the session when the ingest never settles', async () => {+ vi.useFakeTimers();+ finalizeResponse = TIMED_OUT;+ sessionStatus = { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB };const file = fakeFile(40 * MB);- await expect(uploadFileInSlices(file, new Map(), {}))- .rejects.toThrow('HTTP 524');+ const upload = uploadFileInSlices(file, new Map(), {});+ const assertion = expect(upload).rejects.toThrow('HTTP 524');+ await vi.advanceTimersByTimeAsync(11 * 60 * 1000);+ await assertion;expect(deleteCalls()).toHaveLength(0);expect(rememberedSessions()[`talk.mp4|${40 * MB}|0`]).toMatchObject({ sessionId: 'sess-1' });});- it('keeps the session when another request already holds the finalize claim', async () => {- finalizeResponse = { status: 409, text: JSON.stringify({ error: 'Session is already being finalized' }) };+ it('waits out an ingest that was already running when the file was added again', async () => {+ vi.useFakeTimers();+ const file = fakeFile(40 * MB);+ finalizeResponse = TIMED_OUT;+ sessionStatus = { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB };+ const abandoned = uploadFileInSlices(file, new Map(), {});+ const abandonedAssertion = expect(abandoned).rejects.toThrow('HTTP 524');+ await vi.advanceTimersByTimeAsync(11 * 60 * 1000);+ await abandonedAssertion;- await expect(uploadFileInSlices(fakeFile(40 * MB), new Map(), {}))- .rejects.toThrow('already being finalized');+ sent = [];+ fetchCalls = [];+ finalizeResponse = [{ status: 409, text: JSON.stringify({ error: 'Session is already being finalized' }) }, INGESTED];+ sessionStatus = [+ { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB },+ { kind: 'sliced_upload', status: 'finalized', chunk_count: 3, upload_total_bytes: 40 * MB },+ ];- expect(deleteCalls()).toHaveLength(0);- expect(Object.keys(rememberedSessions())).toHaveLength(1);+ const retry = uploadFileInSlices(file, new Map(), {});+ await vi.advanceTimersByTimeAsync(10000);++ expect(await retry).toMatchObject({ id: 42 });+ expect(createCalls()).toHaveLength(0);+ expect(slicePosts()).toHaveLength(0);});it('retries a slice the edge timed out instead of abandoning the upload', async () => {