Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

feat: resume an interrupted sliced upload instead of restarting it

François Catuhecdc7be7a95dd
diff --git a/docs/admin-guide/recording-sessions.md b/docs/admin-guide/recording-sessions.md
index 41a6cbcf..3f78b1ba 100644
--- a/docs/admin-guide/recording-sessions.md
+++ b/docs/admin-guide/recording-sessions.md
@@ -67,6 +67,27 @@ logs:
- A failed finalize deletes the slices with it. They cannot be reused,
since finalize only accepts a session that is still recording.
+### Resuming
+
+An upload interrupted by the network is not restarted. The browser keeps
+the session id in `localStorage` under the file's identity (name, size,
+last-modified), so adding the same file again continues from the slice
+count the server reports rather than from zero, which is what makes the
+feature usable on a link bad enough to need it.
+
+Only a rejection the server would repeat (an oversize slice, an
+exhausted quota, slices that do not add up) deletes the session, because
+retrying that costs the user another full transfer for the same answer.
+A dropped connection or a timeout leaves the session alone, including
+when it happens waiting for `finalize-upload`: the server may be
+ingesting at that moment, and the retry gets that recording back.
+
+Operationally this means in-progress sliced uploads sit in
+`UPLOAD_FOLDER/_sessions/` for up to `RECORDING_SESSION_TTL_HOURS`
+waiting for a resume that may never come, and count against
+`RECORDING_SESSION_MAX_BYTES_PER_USER` until the sweep expires them.
+The browser forgets a session after 24 hours regardless.
+
!!! warning "`RECORDING_SESSION_COMMIT_BATCH_SIZE` must stay at 1"
Above 1, the chunk endpoint flushes its bookkeeping without
diff --git a/static/js/modules/db/sliced-file-upload.js b/static/js/modules/db/sliced-file-upload.js
index 8371500a..f289d517 100644
--- a/static/js/modules/db/sliced-file-upload.js
+++ b/static/js/modules/db/sliced-file-upload.js
@@ -22,6 +22,13 @@
* File slices are Blob views, so nothing here holds the file in memory:
* a multi-gigabyte upload costs one slice's worth of buffering in the
* browser's network stack.
+ *
+ * An upload interrupted by the network resumes rather than restarting:
+ * the session id is kept in localStorage under the file's identity
+ * (name, size, lastModified), and re-adding the same file continues from
+ * the slice count the server reports. Only a rejection the server will
+ * repeat (an oversize slice, an exhausted quota, a size mismatch) drops
+ * the session, because retrying it would fail the same way.
*/
import { getUploadCsrfToken, isCsrfRejection } from '../csrf.js';
@@ -38,6 +45,10 @@ const MAX_RETRY_MS = 30000;
const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
+const RESUME_KEY = 'speakr.slicedUploads';
+const RESUME_TTL_MS = 24 * 60 * 60 * 1000;
+const RESUME_MAX_ENTRIES = 20;
+
/** True when the file is large enough that slicing it is worth a session. */
export function shouldSliceUpload(file) {
return !!file && file.size > SLICE_BYTES;
@@ -45,6 +56,93 @@ export function shouldSliceUpload(file) {
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+ * Identity of the file on this machine. `lastModified` is in it so an
+ * edited or re-exported file of the same name and size is not resumed
+ * onto slices of the older one.
+ */
+function fileKey(file) {
+ return `${file.name}|${file.size}|${file.lastModified || 0}`;
+}
+
+/** Open sessions by file key. Best-effort: private mode has no store. */
+function readResumeMemory() {
+ try {
+ const parsed = JSON.parse(localStorage.getItem(RESUME_KEY) || '{}');
+ const fresh = {};
+ for (const [key, entry] of Object.entries(parsed)) {
+ if (entry?.sessionId && Date.now() - (entry.savedAt || 0) < RESUME_TTL_MS) {
+ fresh[key] = entry;
+ }
+ }
+ return fresh;
+ } catch (_) {
+ return {};
+ }
+}
+
+function writeResumeMemory(memory) {
+ try {
+ const entries = Object.entries(memory)
+ .sort((a, b) => (b[1].savedAt || 0) - (a[1].savedAt || 0))
+ .slice(0, RESUME_MAX_ENTRIES);
+ localStorage.setItem(RESUME_KEY, JSON.stringify(Object.fromEntries(entries)));
+ } catch (_) { /* private mode / quota: resuming is a bonus, not a requirement */ }
+}
+
+function rememberSession(file, sessionId, sliceBytes) {
+ const memory = readResumeMemory();
+ memory[fileKey(file)] = { sessionId, sliceBytes, savedAt: Date.now() };
+ writeResumeMemory(memory);
+}
+
+export function forgetSession(file) {
+ const memory = readResumeMemory();
+ delete memory[fileKey(file)];
+ writeResumeMemory(memory);
+}
+
+/**
+ * The server's view of a session we started for this file, when it can
+ * still be continued. Returns `{sessionId, sliceBytes, chunkCount}` or
+ * null, and forgets anything the server no longer recognises.
+ */
+async function resumableSession(file, token) {
+ const remembered = readResumeMemory()[fileKey(file)];
+ if (!remembered) return null;
+
+ let status;
+ try {
+ const response = await fetch(
+ `${SESSION_BASE}/${encodeURIComponent(remembered.sessionId)}`,
+ { credentials: 'same-origin', headers: { 'X-CSRFToken': token } });
+ if (!response.ok) {
+ forgetSession(file);
+ return null;
+ }
+ status = parseJson(await response.text());
+ } catch (error) {
+ console.warn('[SlicedUpload] Could not read the remembered session:', error);
+ return null;
+ }
+
+ const continuable = status
+ && status.kind === 'sliced_upload'
+ && status.upload_total_bytes === file.size
+ && ['recording', 'finalized'].includes(status.status)
+ && remembered.sliceBytes > 0;
+ if (!continuable) {
+ forgetSession(file);
+ return null;
+ }
+
+ return {
+ sessionId: remembered.sessionId,
+ sliceBytes: remembered.sliceBytes,
+ chunkCount: status.chunk_count || 0,
+ };
+}
+
const retryDelay = (attempts) =>
Math.min(MAX_RETRY_MS, BASE_RETRY_MS * Math.pow(2, attempts - 1));
@@ -116,6 +214,10 @@ async function deleteFileSession(sessionId, token) {
* token when the server rejects it. Returns the index the server expects
* next, which is normally `index + 1` but can jump when a 409 reveals
* the server already holds slices whose responses we lost.
+ *
+ * Failures that outlive the retries are marked resumable when nothing
+ * about them says the next attempt would fail too: the transport, and
+ * the statuses a gateway returns while something behind it restarts.
*/
async function sendSlice(sessionId, index, blob, tokenRef, options) {
const url = `${SESSION_BASE}/${encodeURIComponent(sessionId)}/chunks/${index}`;
@@ -130,7 +232,10 @@ async function sendSlice(sessionId, index, blob, tokenRef, options) {
onXhr: options.onXhr,
});
} catch (transportError) {
- if (attempt >= MAX_SLICE_ATTEMPTS) throw transportError;
+ if (attempt >= MAX_SLICE_ATTEMPTS) {
+ transportError.resumable = true;
+ throw transportError;
+ }
await sleep(retryDelay(attempt));
continue;
}
@@ -150,7 +255,11 @@ async function sendSlice(sessionId, index, blob, tokenRef, options) {
const error = new Error(sliceRejectionMessage(index, blob.size, result, body));
error.status = result.status;
- if (!RETRYABLE_STATUSES.has(result.status) || attempt >= MAX_SLICE_ATTEMPTS) throw error;
+ if (!RETRYABLE_STATUSES.has(result.status)) throw error;
+ if (attempt >= MAX_SLICE_ATTEMPTS) {
+ error.resumable = true;
+ throw error;
+ }
await sleep(retryDelay(attempt));
}
}
@@ -169,14 +278,28 @@ function sliceRejectionMessage(index, sliceSize, result, body) {
return body?.error || `Slice ${index} rejected (HTTP ${result.status})`;
}
+/**
+ * Turn the uploaded slices into a recording.
+ *
+ * A transport failure here is marked resumable: the server may be
+ * ingesting at that moment, so the slices have to survive, and a retry
+ * gets back the recording that call produced rather than sending the
+ * file a second time.
+ */
async function finalize(sessionId, formData, tokenRef, options) {
const url = `${SESSION_BASE}/${encodeURIComponent(sessionId)}/finalize-upload`;
for (let attempt = 1; ; attempt++) {
- const result = await xhrPost(url, formData, {
- token: tokenRef.token,
- timeoutMs: options.finalizeTimeoutMs,
- onXhr: options.onXhr,
- });
+ let result;
+ try {
+ result = await xhrPost(url, formData, {
+ token: tokenRef.token,
+ timeoutMs: options.finalizeTimeoutMs,
+ onXhr: options.onXhr,
+ });
+ } catch (transportError) {
+ transportError.resumable = true;
+ throw transportError;
+ }
const body = parseJson(result.text);
if (result.status >= 200 && result.status < 300 && body?.id) return body;
if (isCsrfRejection(result.status, result.text) && attempt === 1) {
@@ -193,23 +316,36 @@ async function finalize(sessionId, formData, tokenRef, options) {
* single-shot path builds, minus the file part). Resolves to the created
* recording, matching what POST /upload returns.
*
+ * An interrupted upload of the same file continues where it stopped. A
+ * failure the server would repeat takes the session down with it; one
+ * that might not (the network, a timeout) leaves it for the next attempt.
+ *
* `onProgress` is called with a 0..1 fraction of bytes accepted by the
* server. `onXhr` receives each in-flight XHR so a caller can cancel.
*/
export async function uploadFileInSlices(file, formData, options = {}) {
const tokenRef = { token: await getUploadCsrfToken() };
- const session = await createFileSession(file, tokenRef.token);
- const sliceBytes = Math.min(SLICE_BYTES, session.max_chunk_bytes || SLICE_BYTES);
const onProgress = options.onProgress || (() => {});
+ const resumed = await resumableSession(file, tokenRef.token);
+ let sessionId = resumed?.sessionId;
+ let sliceBytes = resumed?.sliceBytes;
+ if (!resumed) {
+ const created = await createFileSession(file, tokenRef.token);
+ sessionId = created.session_id;
+ sliceBytes = Math.min(SLICE_BYTES, created.max_chunk_bytes || SLICE_BYTES);
+ rememberSession(file, sessionId, sliceBytes);
+ }
+
try {
- let index = 1;
- let highWaterMark = 0;
+ let index = (resumed?.chunkCount || 0) + 1;
+ let highWaterMark = index;
let stalledResyncs = 0;
+ onProgress(Math.min(1, ((index - 1) * sliceBytes) / file.size));
while ((index - 1) * sliceBytes < file.size) {
const start = (index - 1) * sliceBytes;
const end = Math.min(start + sliceBytes, file.size);
- index = await sendSlice(session.session_id, index, file.slice(start, end), tokenRef, {
+ index = await sendSlice(sessionId, index, file.slice(start, end), tokenRef, {
onXhr: options.onXhr,
onSliceProgress: (loaded) => onProgress(Math.min(1, (start + loaded) / file.size)),
});
@@ -223,12 +359,16 @@ export async function uploadFileInSlices(file, formData, options = {}) {
}
onProgress(Math.min(1, ((index - 1) * sliceBytes) / file.size));
}
- return await finalize(session.session_id, formData, tokenRef, {
+ const recording = await finalize(sessionId, formData, tokenRef, {
finalizeTimeoutMs: computeUploadTimeout(file.size),
onXhr: options.onXhr,
});
+ forgetSession(file);
+ return recording;
} catch (error) {
- await deleteFileSession(session.session_id, tokenRef.token);
+ if (error.resumable) throw error;
+ await deleteFileSession(sessionId, tokenRef.token);
+ forgetSession(file);
throw error;
}
}
diff --git a/static/js/modules/db/sliced-file-upload.test.js b/static/js/modules/db/sliced-file-upload.test.js
index c976fc9d..be506ca1 100644
--- a/static/js/modules/db/sliced-file-upload.test.js
+++ b/static/js/modules/db/sliced-file-upload.test.js
@@ -12,11 +12,12 @@ import { SLICE_BYTES, shouldSliceUpload, uploadFileInSlices } from './sliced-fil
const MB = 1024 * 1024;
-function fakeFile(size, name = 'talk.mp4', type = 'video/mp4') {
+function fakeFile(size, name = 'talk.mp4', type = 'video/mp4', lastModified = 0) {
return {
name,
type,
size,
+ lastModified,
slice: (start, end) => ({ size: end - start, start, end }),
};
}
@@ -25,16 +26,28 @@ let sent;
let sliceResponses;
let finalizeResponse;
let fetchCalls;
+let sessionStatus;
+let defaultSliceResponse;
+let store;
function _installMocks() {
sent = [];
fetchCalls = [];
sliceResponses = [];
+ defaultSliceResponse = { status: 204, text: '' };
finalizeResponse = { status: 202, text: JSON.stringify({ id: 42, title: 'talk' }) };
+ sessionStatus = null;
global.window = { csrfManager: { refreshToken: async () => 'fresh-token' } };
global.document = { querySelector: () => null };
+ store = new Map();
+ global.localStorage = {
+ getItem: (key) => (store.has(key) ? store.get(key) : null),
+ setItem: (key, value) => store.set(key, String(value)),
+ removeItem: (key) => store.delete(key),
+ };
+
global.fetch = vi.fn(async (url, options) => {
fetchCalls.push({ url, options });
if (options.method === 'POST') {
@@ -47,6 +60,13 @@ function _installMocks() {
}),
};
}
+ if (!options.method) {
+ return {
+ status: sessionStatus ? 200 : 404,
+ ok: !!sessionStatus,
+ text: async () => JSON.stringify(sessionStatus || {}),
+ };
+ }
return { status: 204, text: async () => '' };
});
@@ -66,7 +86,7 @@ function _installMocks() {
sent.push(call);
const response = this.url.endsWith('/finalize-upload')
? finalizeResponse
- : (sliceResponses.shift() || { status: 204, text: '' });
+ : (sliceResponses.shift() || defaultSliceResponse);
queueMicrotask(() => {
if (response.networkError) {
this.onerror();
@@ -84,6 +104,9 @@ function _installMocks() {
}
const slicePosts = () => sent.filter((call) => !call.url.endsWith('/finalize-upload'));
+const createCalls = () => fetchCalls.filter((call) => call.options.method === 'POST');
+const deleteCalls = () => fetchCalls.filter((call) => call.options.method === 'DELETE');
+const rememberedSessions = () => JSON.parse(store.get('speakr.slicedUploads') || '{}');
describe('shouldSliceUpload', () => {
it('leaves files that fit one request on the single-shot path', () => {
@@ -102,6 +125,7 @@ describe('uploadFileInSlices', () => {
delete global.XMLHttpRequest;
delete global.window;
delete global.document;
+ delete global.localStorage;
});
it('opens a session for the filename, sends whole slices in order, and finalizes', async () => {
@@ -246,4 +270,139 @@ describe('uploadFileInSlices', () => {
.rejects.toThrow('Slices on disk do not match the session');
expect(fetchCalls[fetchCalls.length - 1].options.method).toBe('DELETE');
});
+
+ it('forgets the session once the upload is finalized', async () => {
+ await uploadFileInSlices(fakeFile(20 * MB), new Map(), {});
+
+ expect(rememberedSessions()).toEqual({});
+ });
+});
+
+describe('resuming an interrupted upload', () => {
+ beforeEach(() => { _installMocks(); });
+ afterEach(() => {
+ vi.restoreAllMocks();
+ vi.useRealTimers();
+ delete global.fetch;
+ delete global.XMLHttpRequest;
+ delete global.window;
+ delete global.document;
+ delete global.localStorage;
+ });
+
+ const interrupt = async (file) => {
+ vi.useFakeTimers();
+ sliceResponses = [{ status: 204, text: '' }];
+ defaultSliceResponse = { networkError: true };
+ const upload = uploadFileInSlices(file, new Map(), {});
+ const assertion = expect(upload).rejects.toThrow('Network error');
+ await vi.advanceTimersByTimeAsync(120000);
+ await assertion;
+ vi.useRealTimers();
+ defaultSliceResponse = { status: 204, text: '' };
+ };
+
+ it('keeps the session and its slices when the network drops', async () => {
+ const file = fakeFile(40 * MB);
+
+ await interrupt(file);
+
+ expect(deleteCalls()).toHaveLength(0);
+ expect(rememberedSessions()[`talk.mp4|${40 * MB}|0`]).toMatchObject({
+ sessionId: 'sess-1',
+ sliceBytes: SLICE_BYTES,
+ });
+ });
+
+ it('keeps the session when a gateway error outlives the retries', async () => {
+ vi.useFakeTimers();
+ sliceResponses = [{ status: 204, text: '' }];
+ defaultSliceResponse = { status: 502, text: 'Bad Gateway' };
+
+ const upload = uploadFileInSlices(fakeFile(40 * MB), new Map(), {});
+ const assertion = expect(upload).rejects.toThrow('HTTP 502');
+ await vi.advanceTimersByTimeAsync(120000);
+ await assertion;
+
+ expect(deleteCalls()).toHaveLength(0);
+ expect(Object.keys(rememberedSessions())).toHaveLength(1);
+ });
+
+ it('sends only the slices the server is missing on the next attempt', async () => {
+ const file = fakeFile(40 * MB);
+ await interrupt(file);
+ sent = [];
+ fetchCalls = [];
+ sessionStatus = {
+ kind: 'sliced_upload',
+ status: 'recording',
+ chunk_count: 1,
+ upload_total_bytes: 40 * MB,
+ };
+
+ const progress = [];
+ const result = await uploadFileInSlices(file, new Map(), {
+ onProgress: (fraction) => progress.push(fraction),
+ });
+
+ expect(createCalls()).toHaveLength(0);
+ expect(slicePosts().map((call) => [call.url, call.body.start])).toEqual([
+ ['/upload/session/sess-1/chunks/2', SLICE_BYTES],
+ ['/upload/session/sess-1/chunks/3', 2 * SLICE_BYTES],
+ ]);
+ expect(progress[0]).toBeCloseTo(SLICE_BYTES / (40 * MB));
+ expect(result).toEqual({ id: 42, title: 'talk' });
+ expect(rememberedSessions()).toEqual({});
+ });
+
+ it('replays the recording when the upload had already been finalized', async () => {
+ const file = fakeFile(40 * MB);
+ await interrupt(file);
+ sent = [];
+ sessionStatus = {
+ kind: 'sliced_upload',
+ status: 'finalized',
+ chunk_count: 3,
+ upload_total_bytes: 40 * MB,
+ };
+ finalizeResponse = {
+ status: 202,
+ text: JSON.stringify({ id: 42, title: 'talk', idempotent_replay: true }),
+ };
+
+ const result = await uploadFileInSlices(file, new Map(), {});
+
+ expect(slicePosts()).toHaveLength(0);
+ expect(result.id).toBe(42);
+ });
+
+ it('starts over when the file is not the one the session was opened for', async () => {
+ await interrupt(fakeFile(40 * MB));
+ sent = [];
+ fetchCalls = [];
+ sessionStatus = {
+ kind: 'sliced_upload',
+ status: 'recording',
+ chunk_count: 1,
+ upload_total_bytes: 40 * MB,
+ };
+
+ await uploadFileInSlices(fakeFile(40 * MB, 'talk.mp4', 'video/mp4', 99), new Map(), {});
+
+ expect(createCalls()).toHaveLength(1);
+ expect(slicePosts()[0].body.start).toBe(0);
+ });
+
+ it('starts over when the server no longer has the session', async () => {
+ const file = fakeFile(40 * MB);
+ await interrupt(file);
+ sent = [];
+ fetchCalls = [];
+ sessionStatus = null;
+
+ await uploadFileInSlices(file, new Map(), {});
+
+ expect(createCalls()).toHaveLength(1);
+ expect(slicePosts()[0].body.start).toBe(0);
+ });
});
diff --git a/tests/test_sliced_file_upload.py b/tests/test_sliced_file_upload.py
index 3f165eef..f5ff0d1f 100644
--- a/tests/test_sliced_file_upload.py
+++ b/tests/test_sliced_file_upload.py
@@ -358,6 +358,50 @@ def test_a_replayed_finalize_upload_returns_the_first_call_s_recording():
shutil.rmtree(upload_folder, ignore_errors=True)
+def test_an_interrupted_sliced_upload_resumes_from_the_slices_already_sent():
+ """What the client needs to resume: the status endpoint has to say the
+ session is a sliced upload of this exact file and how many slices it
+ already holds, and the next slice has to pick up from there."""
+ upload_folder = _tmp_upload_folder()
+ with app.app_context():
+ app.config["UPLOAD_FOLDER"] = upload_folder
+ user = _mk_user("slice_resume")
+ client = app.test_client()
+ _login(client, user)
+
+ payload = _payload(SLICE_BYTES * 3)
+ session_id = _open_session(client, filename="sample.mp3", mime_type="audio/mpeg",
+ total_bytes=len(payload))["session_id"]
+ _send_slices(client, session_id, payload[:SLICE_BYTES])
+
+ status = client.get(f"/upload/session/{session_id}").get_json()
+ assert status["kind"] == "sliced_upload"
+ assert status["upload_total_bytes"] == len(payload)
+ assert status["status"] == "recording"
+ assert status["chunk_count"] == 1
+
+ for index in (2, 3):
+ start = (index - 1) * SLICE_BYTES
+ resumed = client.post(
+ f"/upload/session/{session_id}/chunks/{index}",
+ data=payload[start:start + SLICE_BYTES],
+ content_type="application/octet-stream",
+ )
+ assert resumed.status_code == 204, (index, resumed.data)
+
+ staging = os.path.join(upload_folder, f"stg_{uuid.uuid4().hex[:6]}")
+ with _upload_mocks(staging):
+ response = client.post(f"/upload/session/{session_id}/finalize-upload",
+ data={}, content_type="multipart/form-data")
+
+ assert response.status_code == 202, response.data
+ recording = db.session.get(Recording, response.get_json()["id"])
+ assert recording.file_hash == hashlib.sha256(payload).hexdigest()
+
+ _cleanup(recording, user)
+ shutil.rmtree(upload_folder, ignore_errors=True)
+
+
def test_another_user_cannot_finalize_someone_elses_sliced_upload():
upload_folder = _tmp_upload_folder()
with app.app_context():