Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python
20.8 KB
Raw Blame
/**
 * Sliced upload of a file the user already has on disk.
 *
 * Speakr's normal upload is one multipart POST of the whole file, so a
 * reverse proxy with a body-size limit below the file size (Cloudflare's
 * 100 MB, for instance) rejects it before Speakr sees it. This module
 * sends the same file as fixed-size byte slices through the
 * recording-session endpoints instead, then POSTs the upload form to
 * /finalize-upload, which byte-joins the slices server-side and ingests
 * the result exactly as POST /upload would.
 *
 * SLICE_BYTES mirrors the server's RECORDING_SESSION_MAX_CHUNK_BYTES
 * default; the create response is authoritative, so a deployment that
 * lowered that env var lowers the slice size with it.
 *
 * It deliberately does NOT reuse server-recording-sessions.js's
 * createSession / abortSession: those maintain the recorder's
 * localStorage crash-recovery marker, which a file upload must not
 * touch. The slice POSTs go through XMLHttpRequest rather than fetch so
 * the progress bar moves within a slice and not just between slices.
 *
 * File slices are Blob views, so nothing here holds the file in memory:
 * a multi-gigabyte upload costs one slice's worth of buffering in the
 * browser's network stack.
 *
 * An upload interrupted by the network resumes rather than restarting:
 * the session id is kept in localStorage under the file's identity
 * (name, size, lastModified), and re-adding the same file continues from
 * the slice count the server reports. Only a rejection the server will
 * repeat (an oversize slice, an exhausted quota, a size mismatch) drops
 * the session, because retrying it would fail the same way.
 */

import { getUploadCsrfToken, isCsrfRejection } from '../csrf.js';
import { computeUploadTimeout } from '../utils/upload-timeout.js';

const SESSION_BASE = '/upload/session';

export const SLICE_BYTES = 16 * 1024 * 1024;

const MAX_SLICE_ATTEMPTS = 6;
const MAX_STALLED_RESYNCS = 2;
const BASE_RETRY_MS = 1000;
const MAX_RETRY_MS = 30000;

const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526, 527]);

const INCONCLUSIVE_STATUSES = new Set([408, 429, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526, 527]);

const INGEST_POLL_MS = 3000;
const INGEST_WAIT_MS = 10 * 60 * 1000;
// A `finalizing` session whose heartbeat is this stale has no ingest behind
// it (the worker died, usually a container recreate). The server lets a new
// finalize take it over after three missed beats, 90 s by default; this sits
// past that so the takeover is attempted rather than waited out. If the
// server's window is longer it answers 409, which is resumable, and the
// loop simply keeps waiting. The server stays the authority either way.
const STALE_INGEST_MS = 120 * 1000;

// Session timestamps are naive UTC ISO strings with no zone suffix, which
// Date.parse would read as local time. Pin them to UTC before comparing.
function ageMs(isoTimestamp) {
    if (!isoTimestamp) return 0;
    const pinned = /[zZ]|[+-]\d\d:?\d\d$/.test(isoTimestamp) ? isoTimestamp : isoTimestamp + 'Z';
    const t = Date.parse(pinned);
    return Number.isFinite(t) ? Date.now() - t : 0;
}
const MAX_FINALIZE_ATTEMPTS = 3;

const RESUME_KEY = 'speakr.slicedUploads';
const RESUME_TTL_MS = 24 * 60 * 60 * 1000;
const RESUME_MAX_ENTRIES = 20;

/** True when the file is large enough that slicing it is worth a session. */
export function shouldSliceUpload(file) {
    return !!file && file.size > SLICE_BYTES;
}

const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

/**
 * Identity of the file on this machine. `lastModified` is in it so an
 * edited or re-exported file of the same name and size is not resumed
 * onto slices of the older one.
 */
function fileKey(file) {
    return `${file.name}|${file.size}|${file.lastModified || 0}`;
}

/** Open sessions by file key. Best-effort: private mode has no store. */
function readResumeMemory() {
    try {
        const parsed = JSON.parse(localStorage.getItem(RESUME_KEY) || '{}');
        const fresh = {};
        for (const [key, entry] of Object.entries(parsed)) {
            if (entry?.sessionId && Date.now() - (entry.savedAt || 0) < RESUME_TTL_MS) {
                fresh[key] = entry;
            }
        }
        return fresh;
    } catch (_) {
        return {};
    }
}

function writeResumeMemory(memory) {
    try {
        const entries = Object.entries(memory)
            .sort((a, b) => (b[1].savedAt || 0) - (a[1].savedAt || 0))
            .slice(0, RESUME_MAX_ENTRIES);
        localStorage.setItem(RESUME_KEY, JSON.stringify(Object.fromEntries(entries)));
    } catch (_) { /* private mode / quota: resuming is a bonus, not a requirement */ }
}

function rememberSession(file, sessionId, sliceBytes) {
    const memory = readResumeMemory();
    memory[fileKey(file)] = { sessionId, sliceBytes, savedAt: Date.now() };
    writeResumeMemory(memory);
}

export function forgetSession(file) {
    const memory = readResumeMemory();
    delete memory[fileKey(file)];
    writeResumeMemory(memory);
}

/**
 * The server's view of a session.
 *
 * Null means the server says there is no such session, which is a fact
 * about the session. Every other failure throws, because a status
 * endpoint that cannot be reached says nothing about the upload behind
 * it, and the difference decides whether a caller gives up or waits.
 */
async function readSessionStatus(sessionId, token) {
    const response = await fetch(`${SESSION_BASE}/${encodeURIComponent(sessionId)}`, {
        credentials: 'same-origin',
        headers: { 'X-CSRFToken': token },
    });
    if (response.status === 404 || response.status === 403) return null;
    if (!response.ok) {
        const error = new Error(`Could not read the upload session (HTTP ${response.status})`);
        error.status = response.status;
        throw error;
    }
    const session = parseJson(await response.text());
    if (!session) throw new Error('Upload session status was not JSON');
    return session;
}

/**
 * The session we opened for this file, when it can still be continued.
 * Returns `{sessionId, sliceBytes, chunkCount}` or null, and forgets
 * anything the server no longer recognises.
 *
 * `finalizing` counts as continuable: it is what an ingest that outran
 * the proxy's patience looks like from here, and the caller waits it out
 * rather than sending the file again.
 */
async function resumableSession(file, token) {
    const remembered = readResumeMemory()[fileKey(file)];
    if (!remembered) return null;

    let status;
    try {
        status = await readSessionStatus(remembered.sessionId, token);
    } catch (error) {
        console.warn('[SlicedUpload] Could not read the remembered session:', error);
        return null;
    }

    const continuable = status
        && status.kind === 'sliced_upload'
        && status.upload_total_bytes === file.size
        && ['recording', 'finalizing', 'finalized'].includes(status.status)
        && remembered.sliceBytes > 0;
    if (!continuable) {
        forgetSession(file);
        return null;
    }

    return {
        sessionId: remembered.sessionId,
        sliceBytes: remembered.sliceBytes,
        chunkCount: status.chunk_count || 0,
    };
}

const retryDelay = (attempts) =>
    Math.min(MAX_RETRY_MS, BASE_RETRY_MS * Math.pow(2, attempts - 1));

function parseJson(text) {
    try { return JSON.parse(text); } catch (_) { return null; }
}

/**
 * POST via XHR, resolving for every HTTP status so callers can decide
 * what is retryable. Rejects only on a network error or timeout, where
 * there is no status and a retry is the right answer.
 */
function xhrPost(url, body, { token, timeoutMs, onProgress, contentType, onXhr }) {
    return new Promise((resolve, reject) => {
        const xhr = new XMLHttpRequest();
        xhr.open('POST', url);
        if (token) xhr.setRequestHeader('X-CSRFToken', token);
        if (contentType) xhr.setRequestHeader('Content-Type', contentType);
        xhr.timeout = timeoutMs;
        if (onProgress) {
            xhr.upload.onprogress = (e) => {
                if (e.lengthComputable) onProgress(e.loaded, e.total);
            };
        }
        xhr.onload = () => resolve({ status: xhr.status, text: xhr.responseText });
        xhr.onerror = () => reject(new Error('Network error during upload'));
        xhr.ontimeout = () => reject(new Error('Upload timed out'));
        xhr.onabort = () => reject(new Error('Upload cancelled'));
        if (onXhr) onXhr(xhr);
        xhr.send(body);
    });
}

async function createFileSession(file, token) {
    const response = await fetch(SESSION_BASE, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json', 'X-CSRFToken': token },
        credentials: 'same-origin',
        body: JSON.stringify({
            filename: file.name,
            mime_type: file.type || '',
            total_bytes: file.size,
        }),
    });
    const body = parseJson(await response.text());
    if (response.status !== 201 || !body?.session_id) {
        throw new Error(body?.error || `Could not open upload session (HTTP ${response.status})`);
    }
    if (!body.upload_filename) {
        throw new Error('Server does not support sliced uploads');
    }
    return body;
}

async function deleteFileSession(sessionId, token) {
    try {
        await fetch(`${SESSION_BASE}/${encodeURIComponent(sessionId)}`, {
            method: 'DELETE',
            credentials: 'same-origin',
            headers: { 'X-CSRFToken': token },
        });
    } catch (error) {
        console.warn('[SlicedUpload] Could not abort session:', error);
    }
}

/**
 * Send one slice, retrying transport failures and refreshing the CSRF
 * token when the server rejects it. Returns the index the server expects
 * next, which is normally `index + 1` but can jump when a 409 reveals
 * the server already holds slices whose responses we lost.
 *
 * Failures that outlive the retries are marked resumable when nothing
 * about them says the next attempt would fail too: the transport, and
 * the statuses a gateway returns while something behind it restarts.
 */
/**
 * How many slices the server already holds for a session, or null when the
 * status could not be read. Null is deliberately not zero: a failed read must
 * fall back to a retry, not be mistaken for "the server has nothing".
 */
async function slicesHeldByServer(sessionId, token) {
    try {
        const status = await readSessionStatus(sessionId, token);
        const n = status && status.chunk_count;
        return Number.isInteger(n) ? n : null;
    } catch (_) {
        return null;
    }
}

async function sendSlice(sessionId, index, blob, tokenRef, options) {
    const url = `${SESSION_BASE}/${encodeURIComponent(sessionId)}/chunks/${index}`;
    for (let attempt = 1; ; attempt++) {
        let result;
        try {
            result = await xhrPost(url, blob, {
                token: tokenRef.token,
                timeoutMs: computeUploadTimeout(blob.size),
                contentType: 'application/octet-stream',
                onProgress: options.onSliceProgress,
                onXhr: options.onXhr,
            });
        } catch (transportError) {
            // A transport failure does not mean the slice was lost. The
            // common case is the opposite: the server took it and answered,
            // and the answer never arrived. That includes a 409 the server
            // sends before it has read the whole body: under HTTP/2 that
            // resets the stream, and the browser reports a protocol error
            // instead of a response we could read `expected_chunk_index`
            // from. Blindly re-sending then draws the same 409 every time
            // until the attempt cap and fails an upload the server has
            // already accepted in full. So ask the server what it holds; a
            // GET has no body to cut short.
            const held = await slicesHeldByServer(sessionId, tokenRef.token);
            if (held !== null && held >= index) return held + 1;

            if (attempt >= MAX_SLICE_ATTEMPTS) {
                transportError.resumable = true;
                throw transportError;
            }
            await sleep(retryDelay(attempt));
            continue;
        }

        if (result.status === 204) return index + 1;

        const body = parseJson(result.text);

        if (result.status === 409 && Number.isInteger(body?.expected_chunk_index)) {
            return body.expected_chunk_index;
        }

        if (isCsrfRejection(result.status, result.text) && attempt < MAX_SLICE_ATTEMPTS) {
            tokenRef.token = await getUploadCsrfToken();
            continue;
        }

        const error = new Error(sliceRejectionMessage(index, blob.size, result, body));
        error.status = result.status;
        if (!RETRYABLE_STATUSES.has(result.status)) throw error;
        if (attempt >= MAX_SLICE_ATTEMPTS) {
            error.resumable = true;
            throw error;
        }
        await sleep(retryDelay(attempt));
    }
}

/**
 * A 413 on a slice is the reverse proxy, not Speakr: its body limit is
 * below the slice size. Naming it the way a proxy does lets the upload
 * composable's error mapping give the user the client_max_body_size
 * guidance instead of a generic processing error.
 */
function sliceRejectionMessage(index, sliceSize, result, body) {
    if (result.status === 413 && !body?.error) {
        const megabytes = Math.round(sliceSize / (1024 * 1024));
        return `Request entity too large: the reverse proxy rejected a ${megabytes} MB upload slice`;
    }
    return body?.error || `Slice ${index} rejected (HTTP ${result.status})`;
}

/**
 * True when the session has to outlive a failed finalize.
 *
 * Covers the statuses that leave the outcome unknown, and the 409 that
 * says another request holds the finalize claim. Cloudflare answers 524
 * at 125 seconds while the origin carries on ingesting, which is the
 * case that matters: the retry is then the replay that hands back the
 * recording rather than a second upload of the same file.
 *
 * Guessing wrong in this direction is cheap. The resume check reads the
 * session's real state first and only continues one the server still
 * has, so a session that did fail costs one request to discover it.
 */
function finalizeCouldStillLand(status) {
    return INCONCLUSIVE_STATUSES.has(status) || status === 409;
}

/**
 * Turn the uploaded slices into a recording.
 *
 * A transport failure or an inconclusive status here is marked resumable:
 * the server may be ingesting at that moment, so the slices have to
 * survive, and a retry gets back the recording that call produced rather
 * than sending the file a second time.
 */
async function finalize(sessionId, formData, tokenRef, options) {
    const url = `${SESSION_BASE}/${encodeURIComponent(sessionId)}/finalize-upload`;
    for (let attempt = 1; ; attempt++) {
        let result;
        try {
            result = await xhrPost(url, formData, {
                token: tokenRef.token,
                timeoutMs: options.finalizeTimeoutMs,
                onXhr: options.onXhr,
            });
        } catch (transportError) {
            transportError.resumable = true;
            throw transportError;
        }
        const body = parseJson(result.text);
        if (result.status >= 200 && result.status < 300 && (body?.id || body?.join_pending)) return body;
        if (isCsrfRejection(result.status, result.text) && attempt === 1) {
            tokenRef.token = await getUploadCsrfToken();
            continue;
        }
        const error = new Error(body?.error
            || `Upload could not be finalized (HTTP ${result.status})`);
        error.status = result.status;
        error.code = body?.code;
        error.resumable = finalizeCouldStillLand(result.status);
        throw error;
    }
}

/**
 * Settle a finalize whose answer never arrived.
 *
 * The ingest runs in-request, so Cloudflare's 125-second Proxy Read
 * Timeout can cut the answer off while the origin carries on and
 * produces the recording anyway. Rather than report a failure the user
 * has to interpret, wait the session out: once it leaves `finalizing`,
 * asking again either replays the recording that landed (`finalized`) or
 * finalizes for real (`recording`, meaning the request never reached
 * Speakr at all). Only when the wait runs out does `firstError` surface,
 * with the session left in place for another attempt.
 *
 * A status endpoint that answers with anything but "no such session" is
 * treated as part of the same outage: this code runs precisely where a
 * proxy is having a bad time, so ending the wait on the first 502 from
 * it would abandon an ingest that is still running.
 */
async function settleUnfinishedFinalize(sessionId, formData, tokenRef, options, firstError) {
    const deadline = Date.now() + INGEST_WAIT_MS;
    let finalizeAttempts = 1;

    while (Date.now() < deadline) {
        let status;
        try {
            status = await readSessionStatus(sessionId, tokenRef.token);
        } catch (_) {
            await sleep(INGEST_POLL_MS);
            continue;
        }

        if (status === null) throw firstError;
        if (status.status === 'finalizing' && ageMs(status.last_seen_at) < STALE_INGEST_MS) {
            // A live ingest is still stamping the row. Wait for it.
            await sleep(INGEST_POLL_MS);
            continue;
        }
        // Either the ingest finished, or it went quiet long enough that the
        // server will hand the session to a fresh finalize. Fall through and
        // try; a 409 means it was not ours to take yet. `finalizing` reaches
        // here only when stale, so it is a legitimate state to retry from.
        if (!['recording', 'finalized', 'finalizing'].includes(status.status)) throw firstError;
        if (finalizeAttempts >= MAX_FINALIZE_ATTEMPTS) throw firstError;

        finalizeAttempts += 1;
        try {
            return await finalize(sessionId, formData, tokenRef, options);
        } catch (error) {
            if (!error.resumable) throw error;
            await sleep(INGEST_POLL_MS);
        }
    }

    throw firstError;
}

/**
 * Upload `file` in slices and finalize with `formData` (the same form the
 * single-shot path builds, minus the file part). Resolves to the created
 * recording, matching what POST /upload returns.
 *
 * An interrupted upload of the same file continues where it stopped. A
 * failure the server would repeat takes the session down with it; one
 * that might not (the network, a timeout) leaves it for the next attempt.
 *
 * `onProgress` is called with a 0..1 fraction of bytes accepted by the
 * server. `onXhr` receives each in-flight XHR so a caller can cancel.
 */
export async function uploadFileInSlices(file, formData, options = {}) {
    const tokenRef = { token: await getUploadCsrfToken() };
    const onProgress = options.onProgress || (() => {});

    const resumed = await resumableSession(file, tokenRef.token);
    let sessionId = resumed?.sessionId;
    let sliceBytes = resumed?.sliceBytes;
    if (!resumed) {
        const created = await createFileSession(file, tokenRef.token);
        sessionId = created.session_id;
        sliceBytes = Math.min(SLICE_BYTES, created.max_chunk_bytes || SLICE_BYTES);
        rememberSession(file, sessionId, sliceBytes);
    }

    try {
        let index = (resumed?.chunkCount || 0) + 1;
        let highWaterMark = index;
        let stalledResyncs = 0;
        onProgress(Math.min(1, ((index - 1) * sliceBytes) / file.size));
        while ((index - 1) * sliceBytes < file.size) {
            const start = (index - 1) * sliceBytes;
            const end = Math.min(start + sliceBytes, file.size);
            index = await sendSlice(sessionId, index, file.slice(start, end), tokenRef, {
                onXhr: options.onXhr,
                onSliceProgress: (loaded) => onProgress(Math.min(1, (start + loaded) / file.size)),
            });
            if (index > highWaterMark) {
                highWaterMark = index;
                stalledResyncs = 0;
            } else if (++stalledResyncs > MAX_STALLED_RESYNCS) {
                throw new Error(
                    `Server keeps asking for slice ${index} after accepting it; `
                    + 'its slice bookkeeping is not advancing');
            }
            onProgress(Math.min(1, ((index - 1) * sliceBytes) / file.size));
        }
        const finalizeOptions = {
            finalizeTimeoutMs: computeUploadTimeout(file.size),
            onXhr: options.onXhr,
        };
        let recording;
        try {
            recording = await finalize(sessionId, formData, tokenRef, finalizeOptions);
        } catch (error) {
            if (!error.resumable) throw error;
            recording = await settleUnfinishedFinalize(
                sessionId, formData, tokenRef, finalizeOptions, error);
        }
        forgetSession(file);
        return recording;
    } catch (error) {
        if (error.resumable) throw error;
        await deleteFileSession(sessionId, tokenRef.token);
        forgetSession(file);
        throw error;
    }
}