Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
feat: resume an interrupted sliced upload instead of restarting it
François Catuhecdc7be7a95dd
diff --git a/docs/admin-guide/recording-sessions.md b/docs/admin-guide/recording-sessions.mdindex 41a6cbcf..3f78b1ba 100644--- a/docs/admin-guide/recording-sessions.md+++ b/docs/admin-guide/recording-sessions.md@@ -67,6 +67,27 @@ logs:- A failed finalize deletes the slices with it. They cannot be reused,since finalize only accepts a session that is still recording.+### Resuming++An upload interrupted by the network is not restarted. The browser keeps+the session id in `localStorage` under the file's identity (name, size,+last-modified), so adding the same file again continues from the slice+count the server reports rather than from zero, which is what makes the+feature usable on a link bad enough to need it.++Only a rejection the server would repeat (an oversize slice, an+exhausted quota, slices that do not add up) deletes the session, because+retrying that costs the user another full transfer for the same answer.+A dropped connection or a timeout leaves the session alone, including+when it happens waiting for `finalize-upload`: the server may be+ingesting at that moment, and the retry gets that recording back.++Operationally this means in-progress sliced uploads sit in+`UPLOAD_FOLDER/_sessions/` for up to `RECORDING_SESSION_TTL_HOURS`+waiting for a resume that may never come, and count against+`RECORDING_SESSION_MAX_BYTES_PER_USER` until the sweep expires them.+The browser forgets a session after 24 hours regardless.+!!! warning "`RECORDING_SESSION_COMMIT_BATCH_SIZE` must stay at 1"Above 1, the chunk endpoint flushes its bookkeeping withoutdiff --git a/static/js/modules/db/sliced-file-upload.js b/static/js/modules/db/sliced-file-upload.jsindex 8371500a..f289d517 100644--- a/static/js/modules/db/sliced-file-upload.js+++ b/static/js/modules/db/sliced-file-upload.js@@ -22,6 +22,13 @@* File slices are Blob views, so nothing here holds the file in memory:* a multi-gigabyte upload costs one slice's worth of buffering in the* browser's network stack.+ *+ * An upload interrupted by the network resumes rather than restarting:+ * the session id is kept in localStorage under the file's identity+ * (name, size, lastModified), and re-adding the same file continues from+ * the slice count the server reports. Only a rejection the server will+ * repeat (an oversize slice, an exhausted quota, a size mismatch) drops+ * the session, because retrying it would fail the same way.*/import { getUploadCsrfToken, isCsrfRejection } from '../csrf.js';@@ -38,6 +45,10 @@ const MAX_RETRY_MS = 30000;const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);+const RESUME_KEY = 'speakr.slicedUploads';+const RESUME_TTL_MS = 24 * 60 * 60 * 1000;+const RESUME_MAX_ENTRIES = 20;+/** True when the file is large enough that slicing it is worth a session. */export function shouldSliceUpload(file) {return !!file && file.size > SLICE_BYTES;@@ -45,6 +56,93 @@ export function shouldSliceUpload(file) {const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));+/**+ * Identity of the file on this machine. `lastModified` is in it so an+ * edited or re-exported file of the same name and size is not resumed+ * onto slices of the older one.+ */+function fileKey(file) {+ return `${file.name}|${file.size}|${file.lastModified || 0}`;+}++/** Open sessions by file key. Best-effort: private mode has no store. */+function readResumeMemory() {+ try {+ const parsed = JSON.parse(localStorage.getItem(RESUME_KEY) || '{}');+ const fresh = {};+ for (const [key, entry] of Object.entries(parsed)) {+ if (entry?.sessionId && Date.now() - (entry.savedAt || 0) < RESUME_TTL_MS) {+ fresh[key] = entry;+ }+ }+ return fresh;+ } catch (_) {+ return {};+ }+}++function writeResumeMemory(memory) {+ try {+ const entries = Object.entries(memory)+ .sort((a, b) => (b[1].savedAt || 0) - (a[1].savedAt || 0))+ .slice(0, RESUME_MAX_ENTRIES);+ localStorage.setItem(RESUME_KEY, JSON.stringify(Object.fromEntries(entries)));+ } catch (_) { /* private mode / quota: resuming is a bonus, not a requirement */ }+}++function rememberSession(file, sessionId, sliceBytes) {+ const memory = readResumeMemory();+ memory[fileKey(file)] = { sessionId, sliceBytes, savedAt: Date.now() };+ writeResumeMemory(memory);+}++export function forgetSession(file) {+ const memory = readResumeMemory();+ delete memory[fileKey(file)];+ writeResumeMemory(memory);+}++/**+ * The server's view of a session we started for this file, when it can+ * still be continued. Returns `{sessionId, sliceBytes, chunkCount}` or+ * null, and forgets anything the server no longer recognises.+ */+async function resumableSession(file, token) {+ const remembered = readResumeMemory()[fileKey(file)];+ if (!remembered) return null;++ let status;+ try {+ const response = await fetch(+ `${SESSION_BASE}/${encodeURIComponent(remembered.sessionId)}`,+ { credentials: 'same-origin', headers: { 'X-CSRFToken': token } });+ if (!response.ok) {+ forgetSession(file);+ return null;+ }+ status = parseJson(await response.text());+ } catch (error) {+ console.warn('[SlicedUpload] Could not read the remembered session:', error);+ return null;+ }++ const continuable = status+ && status.kind === 'sliced_upload'+ && status.upload_total_bytes === file.size+ && ['recording', 'finalized'].includes(status.status)+ && remembered.sliceBytes > 0;+ if (!continuable) {+ forgetSession(file);+ return null;+ }++ return {+ sessionId: remembered.sessionId,+ sliceBytes: remembered.sliceBytes,+ chunkCount: status.chunk_count || 0,+ };+}+const retryDelay = (attempts) =>Math.min(MAX_RETRY_MS, BASE_RETRY_MS * Math.pow(2, attempts - 1));@@ -116,6 +214,10 @@ async function deleteFileSession(sessionId, token) {* token when the server rejects it. Returns the index the server expects* next, which is normally `index + 1` but can jump when a 409 reveals* the server already holds slices whose responses we lost.+ *+ * Failures that outlive the retries are marked resumable when nothing+ * about them says the next attempt would fail too: the transport, and+ * the statuses a gateway returns while something behind it restarts.*/async function sendSlice(sessionId, index, blob, tokenRef, options) {const url = `${SESSION_BASE}/${encodeURIComponent(sessionId)}/chunks/${index}`;@@ -130,7 +232,10 @@ async function sendSlice(sessionId, index, blob, tokenRef, options) {onXhr: options.onXhr,});} catch (transportError) {- if (attempt >= MAX_SLICE_ATTEMPTS) throw transportError;+ if (attempt >= MAX_SLICE_ATTEMPTS) {+ transportError.resumable = true;+ throw transportError;+ }await sleep(retryDelay(attempt));continue;}@@ -150,7 +255,11 @@ async function sendSlice(sessionId, index, blob, tokenRef, options) {const error = new Error(sliceRejectionMessage(index, blob.size, result, body));error.status = result.status;- if (!RETRYABLE_STATUSES.has(result.status) || attempt >= MAX_SLICE_ATTEMPTS) throw error;+ if (!RETRYABLE_STATUSES.has(result.status)) throw error;+ if (attempt >= MAX_SLICE_ATTEMPTS) {+ error.resumable = true;+ throw error;+ }await sleep(retryDelay(attempt));}}@@ -169,14 +278,28 @@ function sliceRejectionMessage(index, sliceSize, result, body) {return body?.error || `Slice ${index} rejected (HTTP ${result.status})`;}+/**+ * Turn the uploaded slices into a recording.+ *+ * A transport failure here is marked resumable: the server may be+ * ingesting at that moment, so the slices have to survive, and a retry+ * gets back the recording that call produced rather than sending the+ * file a second time.+ */async function finalize(sessionId, formData, tokenRef, options) {const url = `${SESSION_BASE}/${encodeURIComponent(sessionId)}/finalize-upload`;for (let attempt = 1; ; attempt++) {- const result = await xhrPost(url, formData, {- token: tokenRef.token,- timeoutMs: options.finalizeTimeoutMs,- onXhr: options.onXhr,- });+ let result;+ try {+ result = await xhrPost(url, formData, {+ token: tokenRef.token,+ timeoutMs: options.finalizeTimeoutMs,+ onXhr: options.onXhr,+ });+ } catch (transportError) {+ transportError.resumable = true;+ throw transportError;+ }const body = parseJson(result.text);if (result.status >= 200 && result.status < 300 && body?.id) return body;if (isCsrfRejection(result.status, result.text) && attempt === 1) {@@ -193,23 +316,36 @@ async function finalize(sessionId, formData, tokenRef, options) {* single-shot path builds, minus the file part). Resolves to the created* recording, matching what POST /upload returns.*+ * An interrupted upload of the same file continues where it stopped. A+ * failure the server would repeat takes the session down with it; one+ * that might not (the network, a timeout) leaves it for the next attempt.+ ** `onProgress` is called with a 0..1 fraction of bytes accepted by the* server. `onXhr` receives each in-flight XHR so a caller can cancel.*/export async function uploadFileInSlices(file, formData, options = {}) {const tokenRef = { token: await getUploadCsrfToken() };- const session = await createFileSession(file, tokenRef.token);- const sliceBytes = Math.min(SLICE_BYTES, session.max_chunk_bytes || SLICE_BYTES);const onProgress = options.onProgress || (() => {});+ const resumed = await resumableSession(file, tokenRef.token);+ let sessionId = resumed?.sessionId;+ let sliceBytes = resumed?.sliceBytes;+ if (!resumed) {+ const created = await createFileSession(file, tokenRef.token);+ sessionId = created.session_id;+ sliceBytes = Math.min(SLICE_BYTES, created.max_chunk_bytes || SLICE_BYTES);+ rememberSession(file, sessionId, sliceBytes);+ }+try {- let index = 1;- let highWaterMark = 0;+ let index = (resumed?.chunkCount || 0) + 1;+ let highWaterMark = index;let stalledResyncs = 0;+ onProgress(Math.min(1, ((index - 1) * sliceBytes) / file.size));while ((index - 1) * sliceBytes < file.size) {const start = (index - 1) * sliceBytes;const end = Math.min(start + sliceBytes, file.size);- index = await sendSlice(session.session_id, index, file.slice(start, end), tokenRef, {+ index = await sendSlice(sessionId, index, file.slice(start, end), tokenRef, {onXhr: options.onXhr,onSliceProgress: (loaded) => onProgress(Math.min(1, (start + loaded) / file.size)),});@@ -223,12 +359,16 @@ export async function uploadFileInSlices(file, formData, options = {}) {}onProgress(Math.min(1, ((index - 1) * sliceBytes) / file.size));}- return await finalize(session.session_id, formData, tokenRef, {+ const recording = await finalize(sessionId, formData, tokenRef, {finalizeTimeoutMs: computeUploadTimeout(file.size),onXhr: options.onXhr,});+ forgetSession(file);+ return recording;} catch (error) {- await deleteFileSession(session.session_id, tokenRef.token);+ if (error.resumable) throw error;+ await deleteFileSession(sessionId, tokenRef.token);+ forgetSession(file);throw error;}}diff --git a/static/js/modules/db/sliced-file-upload.test.js b/static/js/modules/db/sliced-file-upload.test.jsindex c976fc9d..be506ca1 100644--- a/static/js/modules/db/sliced-file-upload.test.js+++ b/static/js/modules/db/sliced-file-upload.test.js@@ -12,11 +12,12 @@ import { SLICE_BYTES, shouldSliceUpload, uploadFileInSlices } from './sliced-filconst MB = 1024 * 1024;-function fakeFile(size, name = 'talk.mp4', type = 'video/mp4') {+function fakeFile(size, name = 'talk.mp4', type = 'video/mp4', lastModified = 0) {return {name,type,size,+ lastModified,slice: (start, end) => ({ size: end - start, start, end }),};}@@ -25,16 +26,28 @@ let sent;let sliceResponses;let finalizeResponse;let fetchCalls;+let sessionStatus;+let defaultSliceResponse;+let store;function _installMocks() {sent = [];fetchCalls = [];sliceResponses = [];+ defaultSliceResponse = { status: 204, text: '' };finalizeResponse = { status: 202, text: JSON.stringify({ id: 42, title: 'talk' }) };+ sessionStatus = null;global.window = { csrfManager: { refreshToken: async () => 'fresh-token' } };global.document = { querySelector: () => null };+ store = new Map();+ global.localStorage = {+ getItem: (key) => (store.has(key) ? store.get(key) : null),+ setItem: (key, value) => store.set(key, String(value)),+ removeItem: (key) => store.delete(key),+ };+global.fetch = vi.fn(async (url, options) => {fetchCalls.push({ url, options });if (options.method === 'POST') {@@ -47,6 +60,13 @@ function _installMocks() {}),};}+ if (!options.method) {+ return {+ status: sessionStatus ? 200 : 404,+ ok: !!sessionStatus,+ text: async () => JSON.stringify(sessionStatus || {}),+ };+ }return { status: 204, text: async () => '' };});@@ -66,7 +86,7 @@ function _installMocks() {sent.push(call);const response = this.url.endsWith('/finalize-upload')? finalizeResponse- : (sliceResponses.shift() || { status: 204, text: '' });+ : (sliceResponses.shift() || defaultSliceResponse);queueMicrotask(() => {if (response.networkError) {this.onerror();@@ -84,6 +104,9 @@ function _installMocks() {}const slicePosts = () => sent.filter((call) => !call.url.endsWith('/finalize-upload'));+const createCalls = () => fetchCalls.filter((call) => call.options.method === 'POST');+const deleteCalls = () => fetchCalls.filter((call) => call.options.method === 'DELETE');+const rememberedSessions = () => JSON.parse(store.get('speakr.slicedUploads') || '{}');describe('shouldSliceUpload', () => {it('leaves files that fit one request on the single-shot path', () => {@@ -102,6 +125,7 @@ describe('uploadFileInSlices', () => {delete global.XMLHttpRequest;delete global.window;delete global.document;+ delete global.localStorage;});it('opens a session for the filename, sends whole slices in order, and finalizes', async () => {@@ -246,4 +270,139 @@ describe('uploadFileInSlices', () => {.rejects.toThrow('Slices on disk do not match the session');expect(fetchCalls[fetchCalls.length - 1].options.method).toBe('DELETE');});++ it('forgets the session once the upload is finalized', async () => {+ await uploadFileInSlices(fakeFile(20 * MB), new Map(), {});++ expect(rememberedSessions()).toEqual({});+ });+});++describe('resuming an interrupted upload', () => {+ beforeEach(() => { _installMocks(); });+ afterEach(() => {+ vi.restoreAllMocks();+ vi.useRealTimers();+ delete global.fetch;+ delete global.XMLHttpRequest;+ delete global.window;+ delete global.document;+ delete global.localStorage;+ });++ const interrupt = async (file) => {+ vi.useFakeTimers();+ sliceResponses = [{ status: 204, text: '' }];+ defaultSliceResponse = { networkError: true };+ const upload = uploadFileInSlices(file, new Map(), {});+ const assertion = expect(upload).rejects.toThrow('Network error');+ await vi.advanceTimersByTimeAsync(120000);+ await assertion;+ vi.useRealTimers();+ defaultSliceResponse = { status: 204, text: '' };+ };++ it('keeps the session and its slices when the network drops', async () => {+ const file = fakeFile(40 * MB);++ await interrupt(file);++ expect(deleteCalls()).toHaveLength(0);+ expect(rememberedSessions()[`talk.mp4|${40 * MB}|0`]).toMatchObject({+ sessionId: 'sess-1',+ sliceBytes: SLICE_BYTES,+ });+ });++ it('keeps the session when a gateway error outlives the retries', async () => {+ vi.useFakeTimers();+ sliceResponses = [{ status: 204, text: '' }];+ defaultSliceResponse = { status: 502, text: 'Bad Gateway' };++ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});+ const assertion = expect(upload).rejects.toThrow('HTTP 502');+ await vi.advanceTimersByTimeAsync(120000);+ await assertion;++ expect(deleteCalls()).toHaveLength(0);+ expect(Object.keys(rememberedSessions())).toHaveLength(1);+ });++ it('sends only the slices the server is missing on the next attempt', async () => {+ const file = fakeFile(40 * MB);+ await interrupt(file);+ sent = [];+ fetchCalls = [];+ sessionStatus = {+ kind: 'sliced_upload',+ status: 'recording',+ chunk_count: 1,+ upload_total_bytes: 40 * MB,+ };++ const progress = [];+ const result = await uploadFileInSlices(file, new Map(), {+ onProgress: (fraction) => progress.push(fraction),+ });++ expect(createCalls()).toHaveLength(0);+ expect(slicePosts().map((call) => [call.url, call.body.start])).toEqual([+ ['/upload/session/sess-1/chunks/2', SLICE_BYTES],+ ['/upload/session/sess-1/chunks/3', 2 * SLICE_BYTES],+ ]);+ expect(progress[0]).toBeCloseTo(SLICE_BYTES / (40 * MB));+ expect(result).toEqual({ id: 42, title: 'talk' });+ expect(rememberedSessions()).toEqual({});+ });++ it('replays the recording when the upload had already been finalized', async () => {+ const file = fakeFile(40 * MB);+ await interrupt(file);+ sent = [];+ sessionStatus = {+ kind: 'sliced_upload',+ status: 'finalized',+ chunk_count: 3,+ upload_total_bytes: 40 * MB,+ };+ finalizeResponse = {+ status: 202,+ text: JSON.stringify({ id: 42, title: 'talk', idempotent_replay: true }),+ };++ const result = await uploadFileInSlices(file, new Map(), {});++ expect(slicePosts()).toHaveLength(0);+ expect(result.id).toBe(42);+ });++ it('starts over when the file is not the one the session was opened for', async () => {+ await interrupt(fakeFile(40 * MB));+ sent = [];+ fetchCalls = [];+ sessionStatus = {+ kind: 'sliced_upload',+ status: 'recording',+ chunk_count: 1,+ upload_total_bytes: 40 * MB,+ };++ await uploadFileInSlices(fakeFile(40 * MB, 'talk.mp4', 'video/mp4', 99), new Map(), {});++ expect(createCalls()).toHaveLength(1);+ expect(slicePosts()[0].body.start).toBe(0);+ });++ it('starts over when the server no longer has the session', async () => {+ const file = fakeFile(40 * MB);+ await interrupt(file);+ sent = [];+ fetchCalls = [];+ sessionStatus = null;++ await uploadFileInSlices(file, new Map(), {});++ expect(createCalls()).toHaveLength(1);+ expect(slicePosts()[0].body.start).toBe(0);+ });});diff --git a/tests/test_sliced_file_upload.py b/tests/test_sliced_file_upload.pyindex 3f165eef..f5ff0d1f 100644--- a/tests/test_sliced_file_upload.py+++ b/tests/test_sliced_file_upload.py@@ -358,6 +358,50 @@ def test_a_replayed_finalize_upload_returns_the_first_call_s_recording():shutil.rmtree(upload_folder, ignore_errors=True)+def test_an_interrupted_sliced_upload_resumes_from_the_slices_already_sent():+ """What the client needs to resume: the status endpoint has to say the+ session is a sliced upload of this exact file and how many slices it+ already holds, and the next slice has to pick up from there."""+ upload_folder = _tmp_upload_folder()+ with app.app_context():+ app.config["UPLOAD_FOLDER"] = upload_folder+ user = _mk_user("slice_resume")+ client = app.test_client()+ _login(client, user)++ payload = _payload(SLICE_BYTES * 3)+ session_id = _open_session(client, filename="sample.mp3", mime_type="audio/mpeg",+ total_bytes=len(payload))["session_id"]+ _send_slices(client, session_id, payload[:SLICE_BYTES])++ status = client.get(f"/upload/session/{session_id}").get_json()+ assert status["kind"] == "sliced_upload"+ assert status["upload_total_bytes"] == len(payload)+ assert status["status"] == "recording"+ assert status["chunk_count"] == 1++ for index in (2, 3):+ start = (index - 1) * SLICE_BYTES+ resumed = client.post(+ f"/upload/session/{session_id}/chunks/{index}",+ data=payload[start:start + SLICE_BYTES],+ content_type="application/octet-stream",+ )+ assert resumed.status_code == 204, (index, resumed.data)++ staging = os.path.join(upload_folder, f"stg_{uuid.uuid4().hex[:6]}")+ with _upload_mocks(staging):+ response = client.post(f"/upload/session/{session_id}/finalize-upload",+ data={}, content_type="multipart/form-data")++ assert response.status_code == 202, response.data+ recording = db.session.get(Recording, response.get_json()["id"])+ assert recording.file_hash == hashlib.sha256(payload).hexdigest()++ _cleanup(recording, user)+ shutil.rmtree(upload_folder, ignore_errors=True)++def test_another_user_cannot_finalize_someone_elses_sliced_upload():upload_folder = _tmp_upload_folder()with app.app_context():