Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

fix: wait out an ingest that outran the proxy instead of reporting failure

François Catuhe459169ea7bbd
diff --git a/docs/admin-guide/recording-sessions.md b/docs/admin-guide/recording-sessions.md
index 67a28e44..e96fa9d2 100644
--- a/docs/admin-guide/recording-sessions.md
+++ b/docs/admin-guide/recording-sessions.md
@@ -92,12 +92,19 @@ ingesting at that moment, and the retry gets that recording back.
the origin finishes ingesting anyway.
Speakr treats 524 and the rest of Cloudflare's 52x range as
- inconclusive rather than fatal, so the session survives and the
- user's retry replays the recording that landed instead of sending
- the file again. Time your worst realistic upload before putting the
- edge in front of it: if it breaches the budget, the fix is deferring
- conversion to the job queue, which `VIDEO_RETENTION=true` already
- does for videos whose original you want to keep.
+ inconclusive rather than fatal. The browser polls the session until
+ the ingest settles, for up to ten minutes, and then either takes the
+ recording that landed or finalizes for real if the request never
+ arrived. Nothing is re-uploaded and the user is not asked to retry;
+ the upload simply takes as long as the ingest does. Re-adding the
+ same file while an ingest is still running waits on it too, rather
+ than starting a second copy.
+
+ Time your worst realistic upload before putting the edge in front of
+ it anyway: a wait that outlives ten minutes still surfaces as a
+ failure, and the fix for that is deferring conversion to the job
+ queue, which `VIDEO_RETENTION=true` already does for videos whose
+ original you want to keep.
Operationally this means in-progress sliced uploads sit in
`UPLOAD_FOLDER/_sessions/` for up to `RECORDING_SESSION_TTL_HOURS`
diff --git a/static/js/modules/db/sliced-file-upload.js b/static/js/modules/db/sliced-file-upload.js
index 8e60f0f2..01eaf970 100644
--- a/static/js/modules/db/sliced-file-upload.js
+++ b/static/js/modules/db/sliced-file-upload.js
@@ -47,6 +47,10 @@ const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504, 520, 521,
const INCONCLUSIVE_STATUSES = new Set([408, 429, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526, 527]);
+const INGEST_POLL_MS = 3000;
+const INGEST_WAIT_MS = 10 * 60 * 1000;
+const MAX_FINALIZE_ATTEMPTS = 3;
+
const RESUME_KEY = 'speakr.slicedUploads';
const RESUME_TTL_MS = 24 * 60 * 60 * 1000;
const RESUME_MAX_ENTRIES = 20;
@@ -104,10 +108,24 @@ export function forgetSession(file) {
writeResumeMemory(memory);
}
+/** The server's view of a session, or null when it no longer has one. */
+async function readSessionStatus(sessionId, token) {
+ const response = await fetch(`${SESSION_BASE}/${encodeURIComponent(sessionId)}`, {
+ credentials: 'same-origin',
+ headers: { 'X-CSRFToken': token },
+ });
+ if (!response.ok) return null;
+ return parseJson(await response.text());
+}
+
/**
- * The server's view of a session we started for this file, when it can
- * still be continued. Returns `{sessionId, sliceBytes, chunkCount}` or
- * null, and forgets anything the server no longer recognises.
+ * The session we opened for this file, when it can still be continued.
+ * Returns `{sessionId, sliceBytes, chunkCount, status}` or null, and
+ * forgets anything the server no longer recognises.
+ *
+ * `finalizing` counts as continuable: it is what an ingest that outran
+ * the proxy's patience looks like from here, and the caller waits it out
+ * rather than sending the file again.
*/
async function resumableSession(file, token) {
const remembered = readResumeMemory()[fileKey(file)];
@@ -115,14 +133,7 @@ async function resumableSession(file, token) {
let status;
try {
- const response = await fetch(
- `${SESSION_BASE}/${encodeURIComponent(remembered.sessionId)}`,
- { credentials: 'same-origin', headers: { 'X-CSRFToken': token } });
- if (!response.ok) {
- forgetSession(file);
- return null;
- }
- status = parseJson(await response.text());
+ status = await readSessionStatus(remembered.sessionId, token);
} catch (error) {
console.warn('[SlicedUpload] Could not read the remembered session:', error);
return null;
@@ -131,7 +142,7 @@ async function resumableSession(file, token) {
const continuable = status
&& status.kind === 'sliced_upload'
&& status.upload_total_bytes === file.size
- && ['recording', 'finalized'].includes(status.status)
+ && ['recording', 'finalizing', 'finalized'].includes(status.status)
&& remembered.sliceBytes > 0;
if (!continuable) {
forgetSession(file);
@@ -142,6 +153,7 @@ async function resumableSession(file, token) {
sessionId: remembered.sessionId,
sliceBytes: remembered.sliceBytes,
chunkCount: status.chunk_count || 0,
+ status: status.status,
};
}
@@ -333,6 +345,48 @@ async function finalize(sessionId, formData, tokenRef, options) {
}
}
+/**
+ * Settle a finalize whose answer never arrived.
+ *
+ * The ingest runs in-request, so Cloudflare's 125-second Proxy Read
+ * Timeout can cut the answer off while the origin carries on and
+ * produces the recording anyway. Rather than report a failure the user
+ * has to interpret, wait the session out: once it leaves `finalizing`,
+ * asking again either replays the recording that landed (`finalized`) or
+ * finalizes for real (`recording`, meaning the request never reached
+ * Speakr at all). Only when the wait runs out does `firstError` surface,
+ * with the session left in place for another attempt.
+ */
+async function settleUnfinishedFinalize(sessionId, formData, tokenRef, options, firstError) {
+ const deadline = Date.now() + INGEST_WAIT_MS;
+ let finalizeAttempts = 1;
+
+ while (Date.now() < deadline) {
+ let status;
+ try {
+ status = await readSessionStatus(sessionId, tokenRef.token);
+ } catch (_) {
+ status = undefined;
+ }
+
+ if (status === null) throw firstError;
+
+ if (status?.status === 'finalizing' || status === undefined) {
+ await sleep(INGEST_POLL_MS);
+ continue;
+ }
+
+ if (['recording', 'finalized'].includes(status.status)
+ && ++finalizeAttempts <= MAX_FINALIZE_ATTEMPTS) {
+ return await finalize(sessionId, formData, tokenRef, options);
+ }
+
+ throw firstError;
+ }
+
+ throw firstError;
+}
+
/**
* Upload `file` in slices and finalize with `formData` (the same form the
* single-shot path builds, minus the file part). Resolves to the created
@@ -381,10 +435,18 @@ export async function uploadFileInSlices(file, formData, options = {}) {
}
onProgress(Math.min(1, ((index - 1) * sliceBytes) / file.size));
}
- const recording = await finalize(sessionId, formData, tokenRef, {
+ const finalizeOptions = {
finalizeTimeoutMs: computeUploadTimeout(file.size),
onXhr: options.onXhr,
- });
+ };
+ let recording;
+ try {
+ recording = await finalize(sessionId, formData, tokenRef, finalizeOptions);
+ } catch (error) {
+ if (!error.resumable) throw error;
+ recording = await settleUnfinishedFinalize(
+ sessionId, formData, tokenRef, finalizeOptions, error);
+ }
forgetSession(file);
return recording;
} catch (error) {
diff --git a/static/js/modules/db/sliced-file-upload.test.js b/static/js/modules/db/sliced-file-upload.test.js
index 0f29b8d9..4fa43da5 100644
--- a/static/js/modules/db/sliced-file-upload.test.js
+++ b/static/js/modules/db/sliced-file-upload.test.js
@@ -61,10 +61,11 @@ function _installMocks() {
};
}
if (!options.method) {
+ const status = nextOf(sessionStatus);
return {
- status: sessionStatus ? 200 : 404,
- ok: !!sessionStatus,
- text: async () => JSON.stringify(sessionStatus || {}),
+ status: status ? 200 : 404,
+ ok: !!status,
+ text: async () => JSON.stringify(status || {}),
};
}
return { status: 204, text: async () => '' };
@@ -85,7 +86,7 @@ function _installMocks() {
const call = { url: this.url, body, headers: this.headers };
sent.push(call);
const response = this.url.endsWith('/finalize-upload')
- ? finalizeResponse
+ ? nextOf(finalizeResponse)
: (sliceResponses.shift() || defaultSliceResponse);
queueMicrotask(() => {
if (response.networkError) {
@@ -103,7 +104,14 @@ function _installMocks() {
};
}
+/** Queued answers are consumed in order; the last one repeats. */
+function nextOf(answers) {
+ if (!Array.isArray(answers)) return answers;
+ return answers.length > 1 ? answers.shift() : answers[0];
+}
+
const slicePosts = () => sent.filter((call) => !call.url.endsWith('/finalize-upload'));
+const finalizePosts = () => sent.filter((call) => call.url.endsWith('/finalize-upload'));
const createCalls = () => fetchCalls.filter((call) => call.options.method === 'POST');
const deleteCalls = () => fetchCalls.filter((call) => call.options.method === 'DELETE');
const rememberedSessions = () => JSON.parse(store.get('speakr.slicedUploads') || '{}');
@@ -314,26 +322,80 @@ describe('resuming an interrupted upload', () => {
});
});
- // Cloudflare answers 524 at 125s while the origin is still ingesting, and the finalize it timed out on may well have landed.
- it('keeps the session when the edge times the finalize out', async () => {
- finalizeResponse = { status: 524, text: '<html><title>524: A timeout occurred</title></html>' };
+ const TIMED_OUT = { status: 524, text: '<html><title>524: A timeout occurred</title></html>' };
+ const INGESTED = { status: 202, text: JSON.stringify({ id: 42, title: 'talk', idempotent_replay: true }) };
+
+ // Cloudflare cuts the answer off at 125s while the origin finishes the ingest, so the recording exists and the client must not re-send the file.
+ it('waits out an ingest the edge timed out on and returns its recording', async () => {
+ vi.useFakeTimers();
+ finalizeResponse = [TIMED_OUT, INGESTED];
+ sessionStatus = [
+ { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB },
+ { kind: 'sliced_upload', status: 'finalized', chunk_count: 3, upload_total_bytes: 40 * MB },
+ ];
+
+ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});
+ await vi.advanceTimersByTimeAsync(10000);
+
+ expect(await upload).toMatchObject({ id: 42, idempotent_replay: true });
+ expect(slicePosts()).toHaveLength(3);
+ expect(finalizePosts()).toHaveLength(2);
+ expect(deleteCalls()).toHaveLength(0);
+ expect(rememberedSessions()).toEqual({});
+ });
+
+ it('finalizes for real when the timed-out request never reached Speakr', async () => {
+ vi.useFakeTimers();
+ finalizeResponse = [{ status: 502, text: 'Bad Gateway' },
+ { status: 202, text: JSON.stringify({ id: 43 }) }];
+ sessionStatus = { kind: 'sliced_upload', status: 'recording', chunk_count: 3, upload_total_bytes: 40 * MB };
+
+ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});
+ await vi.advanceTimersByTimeAsync(10000);
+
+ expect(await upload).toMatchObject({ id: 43 });
+ expect(slicePosts()).toHaveLength(3);
+ });
+
+ it('surfaces the timeout and keeps the session when the ingest never settles', async () => {
+ vi.useFakeTimers();
+ finalizeResponse = TIMED_OUT;
+ sessionStatus = { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB };
const file = fakeFile(40 * MB);
- await expect(uploadFileInSlices(file, new Map(), {}))
- .rejects.toThrow('HTTP 524');
+ const upload = uploadFileInSlices(file, new Map(), {});
+ const assertion = expect(upload).rejects.toThrow('HTTP 524');
+ await vi.advanceTimersByTimeAsync(11 * 60 * 1000);
+ await assertion;
expect(deleteCalls()).toHaveLength(0);
expect(rememberedSessions()[`talk.mp4|${40 * MB}|0`]).toMatchObject({ sessionId: 'sess-1' });
});
- it('keeps the session when another request already holds the finalize claim', async () => {
- finalizeResponse = { status: 409, text: JSON.stringify({ error: 'Session is already being finalized' }) };
+ it('waits out an ingest that was already running when the file was added again', async () => {
+ vi.useFakeTimers();
+ const file = fakeFile(40 * MB);
+ finalizeResponse = TIMED_OUT;
+ sessionStatus = { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB };
+ const abandoned = uploadFileInSlices(file, new Map(), {});
+ const abandonedAssertion = expect(abandoned).rejects.toThrow('HTTP 524');
+ await vi.advanceTimersByTimeAsync(11 * 60 * 1000);
+ await abandonedAssertion;
- await expect(uploadFileInSlices(fakeFile(40 * MB), new Map(), {}))
- .rejects.toThrow('already being finalized');
+ sent = [];
+ fetchCalls = [];
+ finalizeResponse = [{ status: 409, text: JSON.stringify({ error: 'Session is already being finalized' }) }, INGESTED];
+ sessionStatus = [
+ { kind: 'sliced_upload', status: 'finalizing', chunk_count: 3, upload_total_bytes: 40 * MB },
+ { kind: 'sliced_upload', status: 'finalized', chunk_count: 3, upload_total_bytes: 40 * MB },
+ ];
- expect(deleteCalls()).toHaveLength(0);
- expect(Object.keys(rememberedSessions())).toHaveLength(1);
+ const retry = uploadFileInSlices(file, new Map(), {});
+ await vi.advanceTimersByTimeAsync(10000);
+
+ expect(await retry).toMatchObject({ id: 42 });
+ expect(createCalls()).toHaveLength(0);
+ expect(slicePosts()).toHaveLength(0);
});
it('retries a slice the edge timed out instead of abandoning the upload', async () => {