Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

25.0 KB
Raw Blame
import { abortableDelay } from './abortableDelay.ts';
import { DEFAULT_BACKEND_PORT } from '../app-config.ts';
import { deploymentMode, type DeploymentMode } from './deploymentMode.ts';
import { isNativeFaultExit } from './nativeExit.ts';
/**
 * backendCrash — frontend bridge to the desktop shell's crash forensics
 * (#941, src-tauri/src/crash.rs).
 *
 * When the backend PROCESS dies (native CUDA abort, OOM kill, DLL crash) the
 * Rust death watcher persists a crash marker (exit code/signal + stderr tail).
 * This module reads it so:
 *   - api/client.ts can replace the vague "Can't reach the local backend"
 *     with the honest story,
 *   - components/BackendCrashNotice.jsx can offer "View crash details",
 *   - utils/bugReport.js can attach the evidence to the GitHub-issue prefill.
 *
 * Outside the Tauri shell (browser dev, Docker, LAN share) the getters fall
 * back to the BACKEND's own run-sentinel forensics (#1164,
 * backend/core/run_sentinel.py): GET /system/last-run-crash reports a
 * previous run that died without a clean shutdown, adapted here to the same
 * CrashMarker shape — so BackendCrashNotice, the apiFetch crash branch, and
 * the bug-report prefill light up in every deployment, not just desktop.
 */

import i18next from 'i18next';
import {
  CSRF_HEADER_NAME,
  LEGACY_API_KEY_STORAGE_KEY,
  getAdminSession,
  isSameOriginApi,
} from '../api/authSession.ts';

export interface BackendCrashMarker {
  /** Unix seconds when the death was detected. */
  ts: number;
  exit_code: number | null;
  signal: number | null;
  /** Human-readable ExitStatus display ("exit status: 134", …). */
  exit_desc: string;
  backend_version: string;
  /** Seconds the backend had been running when it died. */
  uptime_s: number;
  /** Tail of backend_err.log captured at death time (~40 lines). */
  last_stderr: string;
  /** Whether the user already viewed/dismissed this crash. */
  acknowledged: boolean;
}

function inElectron(): boolean {
  return typeof window !== 'undefined' && !!window.voicestudio;
}

// ── Browser/Docker fallback: the backend's run-sentinel record (#1164) ─────

/** Shape of GET /system/last-run-crash's `record` (backend/core/run_sentinel.py). */
export interface LastRunCrashRecord {
  detected_at: number;
  started_at: number | null;
  ended_between: [number, number];
  uptime_hint_s: number | null;
  version: string;
  last_activity: { ts: number | null; kind: string; detail: string | null } | null;
  log_tail: string[];
}

/** Adapt a run-sentinel record to the CrashMarker shape the whole crash UI
 * already speaks. A sentinel can't know an exit code (the process died out
 * from under it), so exit_code/signal are null and exit_desc carries the
 * story; describeCrashExit() falls through to exit_desc for exactly this
 * shape. Exported for unit tests. */
export function _adaptLastRunCrash(
  record: LastRunCrashRecord,
  acknowledged: boolean,
): BackendCrashMarker {
  const activity = record.last_activity;
  const activityLine = activity?.kind
    ? [
        `last activity before the death: ${activity.kind}${activity.detail ? ` (${activity.detail})` : ''}`,
        '',
      ]
    : [];
  return {
    ts: Math.round(record.detected_at || 0),
    exit_code: null,
    signal: null,
    exit_desc: 'process ended uncleanly (previous run)',
    backend_version: record.version || '',
    uptime_s: Math.max(0, Math.round(record.uptime_hint_s ?? 0)),
    last_stderr: [...activityLine, ...(Array.isArray(record.log_tail) ? record.log_tail : [])]
      .join('\n')
      .trim(),
    acknowledged,
  };
}

/** True when this marker is the run SENTINEL — startup noticing the previous
 * run never cleared its marker — rather than an observed process death. The
 * sentinel genuinely does not know whether the previous run crashed: the
 * machine sleeping, a force-quit, a stopped WSL VM or a Docker restart all
 * leave the same trace, and those benign causes are the majority case
 * (#1375). */
export function isSentinelMarker(marker: BackendCrashMarker): boolean {
  return (
    marker.exit_code === null &&
    marker.signal === null &&
    marker.exit_desc === 'process ended uncleanly (previous run)'
  );
}

/** True when the marker carries something a maintainer could act on: a log
 * tail, or at least a concrete exit code/signal. A sentinel with neither
 * produces a bug report whose evidence block is EMPTY — the reporter files it
 * in good faith, nobody can answer it, and it sits open (#1375). The
 * one-click report is gated on this; the notice itself is not. */
export function hasCrashEvidence(marker: BackendCrashMarker): boolean {
  if ((marker.last_stderr || '').trim()) return true;
  return marker.exit_code !== null || marker.signal !== null;
}

const HTTP_FALLBACK_TIMEOUT_MS = 2500;

/** Auth context a non-desktop deployment may need (LAN-share PIN, ephemeral
 * admin session) — mirrors apiFetch's injection. We deliberately do NOT call apiFetch:
 * its give-up path calls back into this module, and its retry cascade would
 * stall the very error message this fallback exists to enrich. */
function _fallbackHeaders(apiBase: string): Record<string, string> {
  const headers: Record<string, string> = {};
  try {
    const pin = sessionStorage.getItem('ov_pin');
    if (pin) headers['X-OmniVoice-Pin'] = pin;
  } catch {
    /* noop */
  }
  try {
    localStorage.removeItem(LEGACY_API_KEY_STORAGE_KEY);
  } catch {
    /* noop */
  }
  const session = getAdminSession(apiBase);
  if (session) headers['Authorization'] = `Bearer ${session.token}`;
  if (isSameOriginApi(apiBase)) headers[CSRF_HEADER_NAME] = '1';
  return headers;
}

/** Best-effort fetch of the backend's own crash record. Fast timeout, every
 * error swallowed to null — when the backend is DOWN this fails instantly
 * and the caller's mode-aware message stands; the record becomes fetchable
 * once the backend is back (next dev restart / Docker restart policy). */
async function fetchLastRunCrash(): Promise<BackendCrashMarker | null> {
  try {
    // Dynamic import: api/client.ts statically imports this module, so a
    // static import back would be a cycle. apiUrl is only needed at call time.
    const { API, apiUrl } = await import('../api/client.ts');
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), HTTP_FALLBACK_TIMEOUT_MS);
    try {
      const res = await fetch(apiUrl('/system/last-run-crash'), {
        signal: controller.signal,
        headers: _fallbackHeaders(API),
        credentials: 'include',
      });
      if (!res.ok) return null;
      const body = (await res.json()) as {
        record: LastRunCrashRecord | null;
        acknowledged: boolean;
      } | null;
      if (!body?.record) return null;
      return _adaptLastRunCrash(body.record, !!body.acknowledged);
    } finally {
      clearTimeout(timer);
    }
  } catch {
    return null;
  }
}

/** Newest crash marker: the shell's (desktop) or the backend run-sentinel's
 * (browser/dev/Docker), or null when nothing ever crashed / nothing answers. */
export async function getLastBackendCrash(): Promise<BackendCrashMarker | null> {
  if (!inElectron()) return fetchLastRunCrash();
  try {
    const crash = (await window.voicestudio.backend.getStatus()).lastCrash;
    if (!crash) return null;
    return {
      ts: Math.round(crash.timestamp / 1000),
      exit_code: crash.exitCode,
      signal: null,
      exit_desc: crash.signal || (crash.exitCode == null ? 'process ended unexpectedly' : ''),
      backend_version: crash.version,
      uptime_s: Math.round(crash.uptimeMs / 1000),
      last_stderr: crash.logTail.join('\n'),
      acknowledged: crash.acknowledged,
    };
  } catch {
    return null;
  }
}

/** Newest crash marker only if the user hasn't acknowledged it yet. */
export async function getUnacknowledgedBackendCrash(): Promise<BackendCrashMarker | null> {
  const marker = await getLastBackendCrash();
  return marker && !marker.acknowledged ? marker : null;
}

/** Wait, briefly, for the shell to record a crash marker.
 *
 * #1119: the desktop shell learns the backend died from a ~2 s POLL — it has
 * to notice the child exit and write the marker. Asking for that marker ONCE,
 * at the instant a request fails, races that poll and loses: we find nothing
 * and conclude "no crash". That conclusion then gets stated as fact to the
 * user, which is how #1802/#1805 came to read "it most likely crashed or was
 * killed mid-request" on a backend that had left no evidence of dying at all.
 *
 * Outside the Tauri shell there is no death watcher to wait for — the
 * run-sentinel record only appears after the backend RESTARTS — so one
 * immediate ask is all the information there is, and a browser/Docker user is
 * never stalled to learn nothing more.
 *
 * `getCrash` and `sleep` are injectable seams so the branch logic is
 * unit-testable without a shell or real time.
 */
export async function awaitBackendCrashMarker(
  getCrash: () => Promise<BackendCrashMarker | null> = getUnacknowledgedBackendCrash,
  opts: {
    waitMs?: number;
    intervalMs?: number;
    sleep?: (ms: number) => Promise<void>;
    signal?: AbortSignal | null;
  } = {},
): Promise<{ crash: BackendCrashMarker | null; unavailable: boolean }> {
  const waitMs = opts.waitMs ?? 8_000;
  const intervalMs = opts.intervalMs ?? 1_000;
  const sleep = opts.sleep ?? ((ms: number) => abortableDelay(ms, opts.signal));
  const deadline = Date.now() + waitMs;
  for (;;) {
    opts.signal?.throwIfAborted();
    let crash: BackendCrashMarker | null = null;
    try {
      crash = await getCrash();
    } catch {
      opts.signal?.throwIfAborted();
      // Forensics unavailable — "no marker" would be a lie, so say so and let
      // the caller keep its own wording rather than assert anything.
      return { crash: null, unavailable: true };
    }
    opts.signal?.throwIfAborted();
    if (crash) return { crash, unavailable: false };
    if (!inElectron()) return { crash: null, unavailable: false };
    if (Date.now() >= deadline) return { crash: null, unavailable: false };
    await sleep(intervalMs);
  }
}

/** Mark the newest crash as seen (the marker itself is retained for reports). */
export async function acknowledgeBackendCrash(): Promise<void> {
  if (!inElectron()) {
    // Browser/dev/Docker: watermark the backend's run-sentinel record.
    try {
      const { API, apiUrl } = await import('../api/client.ts');
      await fetch(apiUrl('/system/last-run-crash/ack'), {
        method: 'POST',
        headers: _fallbackHeaders(API),
        credentials: 'include',
      });
    } catch {
      /* backend unreachable — the notice will simply resurface, which is honest */
    }
    return;
  }
  try {
    await window.voicestudio.backend.acknowledgeCrash();
  } catch {
    /* shell unavailable — nothing to acknowledge */
  }
}

/** "exit code 3221226505" / "signal 6" / the raw ExitStatus display. */
export function describeCrashExit(
  marker: Pick<BackendCrashMarker, 'exit_code' | 'signal' | 'exit_desc'>,
): string {
  if (marker.exit_code != null) return `exit code ${marker.exit_code}`;
  if (marker.signal != null) return `signal ${marker.signal}`;
  return marker.exit_desc || 'unknown exit';
}

/**
 * Likely-cause line for the crash message, branched on HOW the process died.
 *
 * SIGKILL (signal 9) with no stderr is the operating system's memory killer —
 * on a unified-memory Mac that means system RAM, and the old one-size message
 * blamed "VRAM" on machines that have none (audit finding: OS-OOM kills were
 * misattributed). Everything else keeps the small-GPU VRAM guidance, which is
 * the dominant cause for real GPU aborts.
 */
/**
 * True when the process died of a NATIVE fault — a segfault/illegal
 * instruction/abort — rather than memory pressure or an orderly exit.
 *
 * POSIX reports these as signals. Windows has no signals here: the shell sees
 * the raw NTSTATUS as an exit code, and the two shells disagree on how to read
 * it — Rust as a negative i32, Node as the unsigned DWORD. The table used to
 * live here in Rust's representation, so the same access violation was a fault
 * from Tauri and an ordinary non-zero exit from Electron (#2250). It now lives
 * in `utils/nativeExit`, which normalises before matching, and both shells ask
 * the same question of the same table.
 */
export function isNativeFault(marker: Pick<BackendCrashMarker, 'exit_code' | 'signal'>): boolean {
  return isNativeFaultExit({ exitCode: marker.exit_code, signal: marker.signal });
}

/**
 * True when the captured stderr shows the backend dying while IMPORTING its
 * own dependencies — a broken/incomplete Python environment, not a workload.
 *
 * Reported repeatedly (#1282: `import torchaudio` → `from torch.hub import …`
 * exploding 4 s after launch; #1376: transformers' lazy module raising
 * `ModuleNotFoundError: Could not import module 'GenerationMixin'` 28 s in).
 * Both fell through to the VRAM default, which told users whose venv was
 * half-installed to go flush a TTS model. Nothing about the exit code
 * distinguishes these — the traceback is the only evidence, and it was sitting
 * unread in the marker.
 *
 * Deliberately narrow: an ImportError *plus* one of the packages the app
 * cannot run without. A model file that fails to import mid-request is a
 * different animal, and a generic "ImportError" match would swallow it.
 */
const _IMPORT_FAILURE_MARKERS = [
  'modulenotfounderror',
  'importerror',
  'dll load failed',
  'undefined symbol',
];
const _CORE_DEPENDENCIES = [
  'torch',
  'torchaudio',
  'torchvision',
  'transformers',
  'soundfile',
  'numpy',
];

// How much of the captured tail is treated as "this run". backend_err.log is
// appended across runs and the shell captures its last ~40 lines, so a process
// that died before writing 40 lines of its own carries the PREVIOUS run's
// output above its own (greptile). An import traceback stranded up there must
// not diagnose the crash that happened after it — restricting the match to the
// most recent lines means a stale one only counts when the current run wrote
// almost nothing, which is itself the startup death this classifies.
const _RECENT_TAIL_LINES = 20;

export function isBrokenEnvironmentCrash(
  marker: Partial<Pick<BackendCrashMarker, 'last_stderr'>>,
): boolean {
  const raw = (marker.last_stderr || '').trim();
  if (!raw) return false;
  const tail = raw.split('\n').slice(-_RECENT_TAIL_LINES).join('\n').toLowerCase();
  if (!_IMPORT_FAILURE_MARKERS.some((m) => tail.includes(m))) return false;
  return _CORE_DEPENDENCIES.some((d) => tail.includes(d));
}

export function crashCauseHint(
  marker: Pick<BackendCrashMarker, 'exit_code' | 'signal'> &
    Partial<Pick<BackendCrashMarker, 'last_stderr'>>,
): string {
  // #1223: the backend exits 78 (EX_CONFIG) when it could not bind its port.
  // That is not a crash and has nothing to do with memory — the old message
  // sent a user whose real problem was a leftover process off to shrink their
  // ASR model. Keep in sync with _EXIT_PORT_IN_USE in backend/main.py.
  if (marker.exit_code === 78) {
    return i18next.t('errors.crash_port_in_use', {
      defaultValue:
        `The backend could not start because port ${DEFAULT_BACKEND_PORT} is already in use — another copy of ` +
        'VoiceStudio (or an app that claimed that port) is holding it. Quit the other instance ' +
        'and relaunch; if nothing is visibly running, an orphaned backend from a previous ' +
        'session is still holding the port.',
    });
  }
  if (marker.signal === 9) {
    return i18next.t('errors.crash_oom_kill', {
      defaultValue:
        'It was force-killed (signal 9), which usually means the operating system ran out of ' +
        'memory (RAM) and stopped it. Close memory-heavy apps, pick a smaller ASR model in ' +
        'the engine Weights list in Model Catalogue, or flush the TTS model before transcribing.',
    });
  }
  // Ordered deliberately, between the two explicit-fact branches.
  //
  // AFTER signal 9: an OOM kill is an unambiguous fact about THIS process, and
  // a failed import never produces one — so a stale traceback in the captured
  // tail must never outrank it (greptile).
  //
  // BEFORE the native-fault branch: a dependency that will not load takes the
  // process down as an access violation on Windows (a missing dependent DLL),
  // where "update your GPU driver" is just as wrong as the VRAM advice.
  if (isBrokenEnvironmentCrash(marker)) {
    return i18next.t('errors.crash_broken_env', {
      defaultValue:
        'It died while loading its own Python dependencies, so this is not about memory or ' +
        'your GPU — the environment is incomplete or was left half-updated. Use "Clean & Retry" ' +
        'in Settings → Logs → Backend, which rebuilds it from scratch; that repairs it in ' +
        'place, without touching your voices or projects. If it still fails afterwards, the ' +
        'crash details name the exact package that would not import.',
    });
  }
  // A native crash inside the compute stack — the process was executing bad
  // machine code, not slowly exhausting memory. #1275 (Windows 0xC0000005 on an
  // RTX 2080 SUPER) and #1293 (SIGSEGV on Linux) both landed on the VRAM advice
  // below, which sends the user to flush a model that had nothing to do with it.
  // The real causes are a GPU driver that disagrees with the bundled CUDA
  // runtime, or a truncated/corrupt weight file being memory-mapped.
  if (isNativeFault(marker)) {
    // The crash marker records HOW the process died, not which subsystem was
    // running — a segfault during transcription looks identical to one during
    // synthesis. Naming only the TTS escape hatch sent ASR crashes to a fix
    // that leaves the crashing path untouched (Greptile P1), so both isolated
    // engines are offered and the user picks the one they were using.
    return i18next.t('errors.crash_native_fault', {
      defaultValue:
        'It crashed inside the compute stack rather than running out of memory — that points ' +
        'at a GPU driver that does not match the bundled CUDA runtime, or a model file that ' +
        'downloaded incompletely. Update your GPU driver, then re-download the model from ' +
        'the engine Weights list in Model Catalogue (it repairs a partial download in place). If it keeps happening, ' +
        'switch to a crash-isolated engine in Model Catalogue — "VoiceStudio (subprocess)" ' +
        'for synthesis, "Faster-Whisper (crash-isolated subprocess)" for transcription. Those ' +
        'run the model in a separate process, so a crash like this takes down that process ' +
        'instead of the whole backend.',
    });
  }
  return i18next.t('errors.crash_vram_default', {
    defaultValue:
      'On smaller GPUs the usual cause is running out of VRAM while loading the ASR model on ' +
      'top of the TTS model: flush the TTS model first, or pick a smaller ASR model in ' +
      'the engine Weights list in Model Catalogue.',
  });
}

/** Coarse "12 s" / "3 min" / "2 h" age of a marker, for the honest message. */
export function crashAge(marker: Pick<BackendCrashMarker, 'ts'>, nowMs = Date.now()): string {
  const s = Math.max(0, Math.round(nowMs / 1000 - marker.ts));
  if (s < 90) return `${s} s`;
  const min = Math.round(s / 60);
  if (min < 90) return `${min} min`;
  return `${Math.round(min / 60)} h`;
}

/**
 * The honest error for an SSE/stream that died with NO terminal event (#1062).
 *
 * Every long-running stream the backend serves is contract-bound to emit a
 * terminal event before it closes — even on failure (tests/test_dub_transcribe.py
 * ::test_transcribe_stream_never_closes_without_terminal_event). So a stream that
 * simply goes silent did NOT "probably fail to load a model": the backend
 * PROCESS went away underneath it. On smaller GPUs the usual trigger is running
 * out of VRAM while loading the ASR model on top of a resident TTS model, which
 * aborts the process natively rather than raising a catchable Python error.
 *
 * When the desktop shell recorded a crash marker (#941), say what actually
 * happened — exit code, how long ago, and the VRAM next step — and raise the
 * crash notice so "View crash details" is one click away. With no marker (or
 * outside the Tauri shell) the caller's own message stands.
 *
 * `getCrash` is an injectable seam (same idea as services/endpoint_race's
 * injectable probers) so the branch logic is unit-testable without a shell.
 */
/** Is the backend answering right now? Short timeout — this runs on a failure
 *  path and must not add a visible stall. Any error means "cannot tell". */
async function _probeBackendAlive(): Promise<boolean> {
  try {
    const { API, apiUrl } = await import('../api/client.ts');
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), 2000);
    try {
      const res = await fetch(apiUrl('/system/info'), {
        signal: controller.signal,
        headers: _fallbackHeaders(API),
        credentials: 'include',
      });
      return res.ok;
    } finally {
      clearTimeout(timer);
    }
  } catch {
    return false;
  }
}

export async function streamDropError(
  fallbackMessage: string,
  getCrash: () => Promise<BackendCrashMarker | null> = getUnacknowledgedBackendCrash,
  opts: {
    waitMs?: number;
    intervalMs?: number;
    sleep?: (ms: number) => Promise<void>;
    probeAlive?: () => Promise<boolean>;
    mode?: DeploymentMode;
  } = {},
): Promise<Error> {
  // #1119: the shell learns the backend died from a ~2 s POLL — it must notice
  // the child exit and write the crash marker. Asking for that marker ONCE, at
  // the instant the stream drops, races that poll and loses: we found nothing
  // and fell back to the guess ("Likely ASR backend failed to load") even when
  // the backend had in fact just died. That's the same race #1102 fixed for
  // apiFetch, which this path never got. Give the shell time to catch up before
  // believing there was no crash.
  const { crash, unavailable } = await awaitBackendCrashMarker(getCrash, opts);
  if (unavailable) return new Error(fallbackMessage); // don't mask the caller
  if (!crash) {
    // No crash marker — but "no marker" is not "the backend died and we missed
    // it". Outside the Tauri shell there is no death watcher at all, so this
    // branch is where every browser/Docker user lands, and the caller's
    // fallback used to assert a cause on their behalf ("Likely ASR backend
    // failed to load"). #1242 reported exactly that, in `server` mode, with the
    // backend having answered 20 s earlier — so nothing had crashed and nothing
    // had failed to load.
    //
    // Ask instead of assuming. If the backend is still answering, the process
    // did not go away, which rules the caller's guess out: in a served
    // deployment a stream that dies while the server is healthy is
    // characteristically a reverse proxy or load balancer buffering or timing
    // out the SSE connection.
    const probeAlive = opts.probeAlive ?? _probeBackendAlive;
    if (await probeAlive()) {
      // #2108: the proxy diagnosis below is only possible where a proxy can
      // exist. The desktop webview and `bun run dev` talk to 127.0.0.1
      // directly; a drop there is the local connection dying under a step
      // that went byte-silent for minutes — and the backend kept going (the
      // reporter's transcript was saved 20 min after the UI gave up). Say
      // that, and where to look, instead of handing out nginx advice.
      if ((opts.mode ?? deploymentMode()) !== 'server') {
        return new Error(
          i18next.t('errors.stream_cut_backend_alive_local', {
            defaultValue:
              'The stream ended early, but the backend is still running — so it did not crash. ' +
              'The app lost its connection to the backend mid-job; that happens when a long step ' +
              'goes quiet for minutes, and the backend usually finishes the job anyway. Give it a ' +
              'few minutes and reopen the job from its history, or check the backend log ' +
              '(Settings → Logs → Backend) for what it was doing.',
          }),
        );
      }
      return new Error(
        i18next.t('errors.stream_cut_backend_alive', {
          defaultValue:
            'The stream ended early, but the backend is still running — so it did not crash. ' +
            'In a served or containerised setup this is usually a reverse proxy or load balancer ' +
            'buffering or timing out the connection: disable response buffering for this route ' +
            '(nginx: proxy_buffering off; X-Accel-Buffering: no) and raise its read timeout. ' +
            'Running the desktop app directly, or on localhost without a proxy, will confirm it.',
        }),
      );
    }
    return new Error(fallbackMessage);
  }
  try {
    window.dispatchEvent(new CustomEvent('ov:backend-crashed', { detail: crash }));
  } catch {
    /* no window (tests) — the Error below still tells the story */
  }
  return new Error(
    `The local VoiceStudio backend crashed (${describeCrashExit(crash)}) ${crashAge(crash)} ago, ` +
      'which dropped this stream — it is being restarted automatically. Open the crash notice for ' +
      `the error output. ${crashCauseHint(crash)}`,
  );
}