A desktop coding agent with free access to frontier models.
TypeScript
feat(voice): voice-core, a dependency-free dictation engine
Samrat Talukdarc4f8bb88307c
diff --git a/apps/app/src/app/lib/voice-core/file-source.ts b/apps/app/src/app/lib/voice-core/file-source.tsnew file mode 100644index 00000000..02ce26a8--- /dev/null+++ b/apps/app/src/app/lib/voice-core/file-source.ts@@ -0,0 +1,70 @@+import { resampleTo16k } from "./resample";+import { VOICE_SAMPLE_RATE, VoiceError, type AudioSource } from "./types";+import { decodeWav } from "./wav";++/** The test hook: a WAV file played in place of the microphone. */+export const VOICE_INPUT_FILE_ENV = "OMNIRUSH_VOICE_INPUT_FILE";++export function voiceInputFileFromEnv(env: Record<string, string | undefined>): string | null {+ const value = env[VOICE_INPUT_FILE_ENV]?.trim();+ return value ? value : null;+}++/**+ * An AudioSource that plays a WAV file (any PCM format, resampled to 16 kHz+ * mono) in chunks, paced in real time unless `realtime` is false. The caller+ * reads the file; this module never touches a file system.+ */+export class WavFileSource implements AudioSource {+ private readonly pcm: Int16Array;+ private timer: ReturnType<typeof setInterval> | null = null;+ private stopped = false;++ constructor(wavBytes: Uint8Array, private readonly options: { realtime?: boolean; chunkMs?: number } = {}) {+ let decoded;+ try {+ decoded = decodeWav(wavBytes);+ } catch (error) {+ throw new VoiceError("capture_failed", `The voice input file could not be read: ${error instanceof Error ? error.message : String(error)}`);+ }+ this.pcm = resampleTo16k(decoded.samples, decoded.sampleRate);+ }++ get durationMs(): number {+ return (this.pcm.length / VOICE_SAMPLE_RATE) * 1000;+ }++ async start(onFrames: (frames: Int16Array) => void, onEnd?: (error?: VoiceError) => void): Promise<void> {+ this.stopped = false;+ const chunkMs = this.options.chunkMs ?? 100;+ const chunk = Math.round((VOICE_SAMPLE_RATE * chunkMs) / 1000);+ let offset = 0;+ const step = () => {+ if (this.stopped) return false;+ if (offset >= this.pcm.length) {+ this.stop();+ onEnd?.();+ return false;+ }+ onFrames(this.pcm.slice(offset, offset + chunk));+ offset += chunk;+ return true;+ };+ if (this.options.realtime === false) {+ // Delivered after start() resolves, as a microphone would.+ setTimeout(() => {+ while (step()) {+ // drain+ }+ }, 0);+ return;+ }+ this.timer = setInterval(step, chunkMs);+ }++ stop(): void {+ this.stopped = true;+ if (this.timer) clearInterval(this.timer);+ this.timer = null;+ }+}diff --git a/apps/app/src/app/lib/voice-core/index.ts b/apps/app/src/app/lib/voice-core/index.tsnew file mode 100644index 00000000..a5356cca--- /dev/null+++ b/apps/app/src/app/lib/voice-core/index.ts@@ -0,0 +1,21 @@+/**+ * voice-core: dictation without a UI. Capture-agnostic (any AudioSource+ * delivering 16 kHz mono Int16), dependency-free TypeScript that runs in a+ * browser renderer and in Node 20+, so the CLI can vendor this folder as is.+ *+ * source → Segmenter (energy VAD, pause-cut) → encodeWav → Transcriber+ * → ordered stitch → keyterm post-correction → text+ *+ * Audio is disposable: silence is dropped as it arrives, each speech+ * segment's PCM and WAV are released once its text (or final failure) is+ * known, and nothing is ever written to disk.+ */+export * from "./types";+export { encodeWav, decodeWav, type DecodedWav } from "./wav";+export { Downsampler, resampleTo16k } from "./resample";+export { LevelMeter, Segmenter, levelFromRms, rms, type Segment, type SegmenterOptions } from "./segmenter";+export { countWords, isCjk, isLikelyHallucination, padInsertion, stitch } from "./text";+export { DEV_KEYTERMS, applyKeyterms, buildKeyterms, splitIdentifier, type KeytermContext } from "./keyterms";+export { RemoteTranscriber, errorCode, voiceErrorFromResponse, type RemoteTranscriberOptions } from "./transcriber";+export { CircuitBreaker, VoiceSession, type VoiceSessionOptions } from "./session";+export { VOICE_INPUT_FILE_ENV, WavFileSource, voiceInputFileFromEnv } from "./file-source";diff --git a/apps/app/src/app/lib/voice-core/keyterms.ts b/apps/app/src/app/lib/voice-core/keyterms.tsnew file mode 100644index 00000000..cebffb9d--- /dev/null+++ b/apps/app/src/app/lib/voice-core/keyterms.ts@@ -0,0 +1,117 @@+/** Coding vocabulary sent as a hint with every recording. */+export const DEV_KEYTERMS = [+ "OmniRush",+ "TypeScript",+ "JavaScript",+ "JSON",+ "YAML",+ "OAuth",+ "GitHub",+ "localhost",+ "regex",+ "gRPC",+ "API",+ "CLI",+ "MCP",+ "npm",+ "pnpm",+ "worktree",+ "subagent",+ "async",+ "README",+] as const;++const MAX_TERMS = 50;+const MAX_CHARS = 1024;++/** `feat/voice-mode_v2`, `useVoiceDictation`, `voice-core.ts` → their words. */+export function splitIdentifier(value: string): string[] {+ return value+ .replace(/\.[a-z0-9]{1,5}$/i, "")+ .replace(/([a-z0-9])([A-Z])/g, "$1 $2")+ .split(/[^\p{L}\p{N}]+/u)+ .filter((word) => word.length >= 3 && !/^\d+$/.test(word));+}++export type KeytermContext = {+ /** The project folder or repository name. */+ repo?: string | null;+ branch?: string | null;+ /** Recently used file paths or names. */+ files?: readonly string[];+ extra?: readonly string[];+};++/**+ * At most 50 terms and 1024 characters: the dev list, then the repository,+ * branch and file names (whole and split into words), de-duplicated+ * case-insensitively, first occurrence wins.+ */+export function buildKeyterms(context: KeytermContext = {}): string[] {+ const candidates: string[] = [...DEV_KEYTERMS, ...(context.extra ?? [])];+ if (context.repo?.trim()) candidates.push(context.repo.trim(), ...splitIdentifier(context.repo));+ if (context.branch?.trim() && !/^(main|master|HEAD)$/.test(context.branch.trim())) candidates.push(...splitIdentifier(context.branch));+ for (const file of context.files ?? []) {+ const name = file.split(/[\\/]/).pop() ?? "";+ if (!name) continue;+ candidates.push(name, ...splitIdentifier(name));+ }+ const seen = new Set<string>();+ const out: string[] = [];+ let chars = 0;+ for (const raw of candidates) {+ const term = raw.trim();+ const key = term.toLowerCase();+ if (!term || term.length > 64 || seen.has(key)) continue;+ if (out.length >= MAX_TERMS || chars + term.length + 1 > MAX_CHARS) break;+ seen.add(key);+ out.push(term);+ chars += term.length + 1;+ }+ return out;+}++/** Spoken forms of written dev terms. Conservative: only phrases no one says for anything else. */+const SPOKEN_FORMS: ReadonlyArray<[RegExp, string]> = [+ [/\bo(?:h)?[ -]auth\b/gi, "OAuth"],+ [/\bgit[ -]?hub\b/gi, "GitHub"],+ [/\bget[ -]hub\b/gi, "GitHub"],+ [/\btype[ -]script\b/gi, "TypeScript"],+ [/\bjava[ -]script\b/gi, "JavaScript"],+ [/\bnode[ .]?js\b/gi, "Node.js"],+ [/\bnext[ .]?js\b/gi, "Next.js"],+ [/\blocal[ -]host\b/gi, "localhost"],+ [/\bwork[ -]tree(s?)\b/gi, "worktree$1"],+ [/\bsub[ -]agent(s?)\b/gi, "subagent$1"],+ [/\bread[ -]me\b/gi, "README"],+ [/\bp[ -]?npm\b/gi, "pnpm"],+ [/\bomni[ -]rush\b/gi, "OmniRush"],+];++const EXTENSIONS = "ts|tsx|js|jsx|mjs|cjs|py|rs|go|md|json|yaml|yml|toml|css|html|sh|sql|txt";+const DOT_EXTENSION = new RegExp(`\\b([\\p{L}\\p{N}_-]+) dot (${EXTENSIONS})\\b`, "giu");++function escapeRegExp(value: string): string {+ return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");+}++/**+ * Light post-correction of a transcript: spoken dev terms to their written+ * form, "auth dot ts" to "auth.ts", and project terms with inner capitals+ * ("omni rush" / "omnirush" → "OmniRush") restored from the keyterms.+ */+export function applyKeyterms(text: string, keyterms: readonly string[] = []): string {+ let out = text;+ for (const [pattern, replacement] of SPOKEN_FORMS) out = out.replace(pattern, replacement);+ out = out.replace(DOT_EXTENSION, (_match, name: string, extension: string) => `${name}.${extension.toLowerCase()}`);+ for (const term of keyterms) {+ // Only terms whose casing carries meaning, never plain words.+ if (term.length < 4 || term === term.toLowerCase() || /[^\p{L}\p{N}]/u.test(term)) continue;+ const words = splitIdentifier(term);+ // The words must spell the whole term ("gRPC" splits to "RPC": skipped).+ if (words.join("") !== term) continue;+ const spoken = words.map(escapeRegExp).join("[ -]?");+ out = out.replace(new RegExp(`\\b${spoken}\\b`, "gi"), term);+ }+ return out;+}diff --git a/apps/app/src/app/lib/voice-core/resample.ts b/apps/app/src/app/lib/voice-core/resample.tsnew file mode 100644index 00000000..a1186f16--- /dev/null+++ b/apps/app/src/app/lib/voice-core/resample.ts@@ -0,0 +1,63 @@+import { VOICE_SAMPLE_RATE } from "./types";++function toInt16(value: number): number {+ const clamped = Math.max(-1, Math.min(1, value));+ return clamped < 0 ? Math.round(clamped * 32768) : Math.round(clamped * 32767);+}++/**+ * A streaming converter from float audio at `inputRate` to 16 kHz Int16.+ * Downsampling averages every input sample that falls inside an output+ * sample's window (a box filter: cheap, and enough against aliasing for+ * speech); upsampling interpolates linearly. State carries across calls, so+ * chunk boundaries leave no clicks.+ */+export class Downsampler {+ private readonly ratio: number;+ /** Position of the next output sample, in input samples from the start of the pending chunk. */+ private position = 0;+ private carry: Float32Array = new Float32Array(0);++ constructor(readonly inputRate: number, readonly outputRate = VOICE_SAMPLE_RATE) {+ if (!(inputRate > 0) || !(outputRate > 0)) throw new Error("Sample rates must be positive");+ this.ratio = inputRate / outputRate;+ }++ push(input: Float32Array): Int16Array {+ if (this.inputRate === this.outputRate) {+ const out = new Int16Array(input.length);+ for (let index = 0; index < input.length; index += 1) out[index] = toInt16(input[index]!);+ return out;+ }+ const data = new Float32Array(this.carry.length + input.length);+ data.set(this.carry);+ data.set(input, this.carry.length);+ const out: number[] = [];+ if (this.ratio >= 1) {+ while (this.position + this.ratio <= data.length) {+ const start = Math.floor(this.position);+ const end = Math.floor(this.position + this.ratio);+ let sum = 0;+ for (let index = start; index < end; index += 1) sum += data[index]!;+ out.push(toInt16(sum / Math.max(1, end - start)));+ this.position += this.ratio;+ }+ } else {+ while (this.position + 1 < data.length) {+ const index = Math.floor(this.position);+ const fraction = this.position - index;+ out.push(toInt16(data[index]! * (1 - fraction) + data[index + 1]! * fraction));+ this.position += this.ratio;+ }+ }+ const consumed = Math.floor(this.position);+ this.carry = data.slice(consumed);+ this.position -= consumed;+ return Int16Array.from(out);+ }+}++/** Whole-buffer conversion (file input). */+export function resampleTo16k(samples: Float32Array, inputRate: number): Int16Array {+ return new Downsampler(inputRate).push(samples);+}diff --git a/apps/app/src/app/lib/voice-core/segmenter.ts b/apps/app/src/app/lib/voice-core/segmenter.tsnew file mode 100644index 00000000..65eb315f--- /dev/null+++ b/apps/app/src/app/lib/voice-core/segmenter.ts@@ -0,0 +1,231 @@+import { VOICE_SAMPLE_RATE } from "./types";++/** RMS of Int16 samples, normalized to 0..1. */+export function rms(samples: Int16Array): number {+ if (samples.length === 0) return 0;+ let sum = 0;+ for (let index = 0; index < samples.length; index += 1) {+ const value = samples[index]! / 32768;+ sum += value * value;+ }+ return Math.sqrt(sum / samples.length);+}++/** A display level 0..1 for a normalized RMS: square-root scaled so quiet speech still moves the bars. */+export function levelFromRms(value: number): number {+ return Math.min(1, Math.sqrt((value * 32768) / 2000));+}++/** The rolling 16-bar level history the waveform draws. */+export class LevelMeter {+ private readonly history: number[];++ constructor(readonly bars = 16) {+ this.history = new Array<number>(bars).fill(0);+ }++ push(value: number): void {+ this.history.push(levelFromRms(value));+ if (this.history.length > this.bars) this.history.shift();+ }++ levels(): number[] {+ return [...this.history];+ }+}++export type Segment = {+ index: number;+ /** 16 kHz mono PCM, including a short lead-in before the first speech frame. */+ samples: Int16Array;+ startMs: number;+ endMs: number;+ /** Frames classified as speech, in ms. */+ speechMs: number;+ /** Highest frame RMS in the segment, 0..1. */+ peakRms: number;+ /** The speech threshold when the segment closed, for the hallucination filter. */+ threshold: number;+};++export type SegmenterOptions = {+ sampleRate?: number;+ /** Analysis frame length. */+ frameMs?: number;+ /** A pause at least this long ends a segment. */+ pauseMs?: number;+ /** A segment is cut here even mid-speech. */+ maxSegmentMs?: number;+ /** A segment shorter than this is held open across a normal pause (short segments transcribe badly). */+ minSegmentMs?: number;+ /** A pause this long ends even a short segment. */+ longPauseMs?: number;+ /** Less speech than this (a click, a cough) is dropped as noise. */+ minSpeechMs?: number;+ /** Audio kept before the first speech frame, so onsets are not clipped. */+ prerollMs?: number;+ /** Silence kept after the last speech frame. */+ tailMs?: number;+ /** Absolute speech floor (normalized RMS); the adaptive threshold never goes below it. */+ minThreshold?: number;+ /** Speech is this many times above the running noise floor. */+ noiseMultiplier?: number;+};++const DEFAULTS: Required<SegmenterOptions> = {+ sampleRate: VOICE_SAMPLE_RATE,+ frameMs: 20,+ pauseMs: 600,+ maxSegmentMs: 12_000,+ minSegmentMs: 1_000,+ longPauseMs: 1_500,+ minSpeechMs: 120,+ prerollMs: 200,+ tailMs: 250,+ minThreshold: 0.01,+ noiseMultiplier: 3,+};++/**+ * Energy VAD with an adaptive noise floor that splits a PCM stream into+ * speech segments at pauses. Silence outside a segment is discarded as it+ * arrives: a silent recording produces no segment at all, so nothing is ever+ * uploaded for it.+ */+export class Segmenter {+ readonly options: Required<SegmenterOptions>;+ private readonly frameSize: number;+ private pendingFrame: Int16Array = new Int16Array(0);+ private noiseFloor = 0.004;+ private frames: Int16Array[] = [];+ private frameRms: number[] = [];+ /** Frames before the first speech of the next segment, capped at the preroll. */+ private preroll: Int16Array[] = [];+ private inSegment = false;+ private speechFrames = 0;+ private silenceRun = 0;+ private segmentStartFrame = 0;+ private framesSeen = 0;+ private nextIndex = 0;+ /** Highest frame RMS of the whole stream. */+ maxRms = 0;+ /** Speech frames of the whole stream (including dropped noise bursts). */+ totalSpeechFrames = 0;++ constructor(options: SegmenterOptions = {}) {+ this.options = { ...DEFAULTS, ...options };+ this.frameSize = Math.round((this.options.sampleRate * this.options.frameMs) / 1000);+ }++ get threshold(): number {+ return Math.max(this.options.minThreshold, this.noiseFloor * this.options.noiseMultiplier);+ }++ /** Feeds PCM; returns segments that closed. `onFrame` sees every analysis frame's RMS (the level meter). */+ push(samples: Int16Array, onFrame?: (frameRms: number) => void): Segment[] {+ const closed: Segment[] = [];+ let data = samples;+ if (this.pendingFrame.length > 0) {+ data = new Int16Array(this.pendingFrame.length + samples.length);+ data.set(this.pendingFrame);+ data.set(samples, this.pendingFrame.length);+ }+ let offset = 0;+ for (; offset + this.frameSize <= data.length; offset += this.frameSize) {+ const frame = data.slice(offset, offset + this.frameSize);+ const value = rms(frame);+ onFrame?.(value);+ const segment = this.frame(frame, value);+ if (segment) closed.push(segment);+ }+ this.pendingFrame = data.slice(offset);+ return closed;+ }++ /** Ends the stream: the open segment, if it holds enough speech. */+ flush(): Segment | null {+ this.pendingFrame = new Int16Array(0);+ if (!this.inSegment) return null;+ return this.close(true);+ }++ private frame(frame: Int16Array, value: number): Segment | null {+ this.framesSeen += 1;+ this.maxRms = Math.max(this.maxRms, value);+ const speech = value > this.threshold;+ if (speech) this.totalSpeechFrames += 1;+ else this.noiseFloor = Math.min(0.05, Math.max(0.001, this.noiseFloor * 0.95 + value * 0.05));+ const { frameMs } = this.options;++ if (!this.inSegment) {+ if (!speech) {+ this.preroll.push(frame);+ if (this.preroll.length * frameMs > this.options.prerollMs) this.preroll.shift();+ return null;+ }+ this.inSegment = true;+ this.frames = [...this.preroll, frame];+ this.frameRms = [...this.preroll.map(rms), value];+ this.segmentStartFrame = this.framesSeen - this.frames.length;+ this.preroll = [];+ this.speechFrames = 1;+ this.silenceRun = 0;+ return null;+ }++ this.frames.push(frame);+ this.frameRms.push(value);+ if (speech) {+ this.speechFrames += 1;+ this.silenceRun = 0;+ } else {+ this.silenceRun += 1;+ }+ const lengthMs = this.frames.length * frameMs;+ const silenceMs = this.silenceRun * frameMs;+ if (lengthMs >= this.options.maxSegmentMs) return this.close(false);+ if (silenceMs >= this.options.longPauseMs) return this.close(true);+ if (silenceMs >= this.options.pauseMs && lengthMs - silenceMs >= this.options.minSegmentMs) return this.close(true);+ return null;+ }++ private close(trimSilence: boolean): Segment | null {+ const { frameMs } = this.options;+ let frames = this.frames;+ let frameRms = this.frameRms;+ if (trimSilence && this.silenceRun > 0) {+ const keep = Math.ceil(this.options.tailMs / frameMs);+ const drop = Math.max(0, this.silenceRun - keep);+ frames = frames.slice(0, frames.length - drop);+ frameRms = frameRms.slice(0, frameRms.length - drop);+ // The trimmed silence may lead into the next segment's preroll.+ this.preroll = this.frames.slice(this.frames.length - Math.min(drop, Math.ceil(this.options.prerollMs / frameMs)));+ } else {+ this.preroll = [];+ }+ const speechMs = this.speechFrames * frameMs;+ const startFrame = this.segmentStartFrame;+ this.inSegment = false;+ this.frames = [];+ this.frameRms = [];+ this.speechFrames = 0;+ this.silenceRun = 0;+ if (speechMs < this.options.minSpeechMs) return null;+ const length = frames.reduce((total, frame) => total + frame.length, 0);+ const samples = new Int16Array(length);+ let at = 0;+ for (const frame of frames) {+ samples.set(frame, at);+ at += frame.length;+ }+ return {+ index: this.nextIndex++,+ samples,+ startMs: startFrame * frameMs,+ endMs: (startFrame + frames.length) * frameMs,+ speechMs,+ peakRms: Math.max(0, ...frameRms),+ threshold: this.threshold,+ };+ }+}diff --git a/apps/app/src/app/lib/voice-core/session.ts b/apps/app/src/app/lib/voice-core/session.tsnew file mode 100644index 00000000..0b11366b--- /dev/null+++ b/apps/app/src/app/lib/voice-core/session.ts@@ -0,0 +1,399 @@+import { applyKeyterms } from "./keyterms";+import { LevelMeter, Segmenter, type Segment, type SegmenterOptions } from "./segmenter";+import { countWords, isLikelyHallucination, stitch } from "./text";+import {+ VOICE_SAMPLE_RATE,+ VoiceError,+ isVoiceError,+ type AudioSource,+ type Transcriber,+ type VoicePhase,+ type VoiceResult,+ type VoiceSnapshot,+} from "./types";+import { encodeWav } from "./wav";++/** Pauses voice after `threshold` failures within `windowMs`, for `cooldownMs`. Shared by every recording. */+export class CircuitBreaker {+ private failures: number[] = [];+ private openUntil = 0;++ constructor(+ private readonly threshold = 3,+ private readonly windowMs = 10_000,+ private readonly cooldownMs = 30_000,+ private readonly now: () => number = Date.now,+ ) {}++ recordFailure(): void {+ const at = this.now();+ this.failures = [...this.failures.filter((time) => at - time < this.windowMs), at];+ if (this.failures.length >= this.threshold) {+ this.openUntil = at + this.cooldownMs;+ this.failures = [];+ }+ }++ recordSuccess(): void {+ this.failures = [];+ }++ isOpen(): boolean {+ return this.now() < this.openUntil;+ }+}++export type VoiceSessionOptions = {+ source: AudioSource;+ transcriber: Transcriber;+ language?: string | null;+ keyterms?: readonly string[];+ /** Uploads in flight at once. */+ concurrency?: number;+ /** Pauses before each retry of a failed upload; its length is the retry count. */+ retryDelaysMs?: readonly number[];+ /** Hard cap on one recording. */+ maxRecordingMs?: number;+ /** Stop by itself after this much silence (tap mode); null never. */+ silenceAutoStopMs?: number | null;+ /** How long stop() waits for the last uploads. */+ finalizeTimeoutMs?: number;+ /** Below this peak RMS the microphone is reported as delivering no signal. */+ noSignalRms?: number;+ segmenter?: SegmenterOptions;+ breaker?: CircuitBreaker;+ recordingId?: string;+ /** Every state change, and level updates at most every `snapshotIntervalMs`. */+ onChange?: (snapshot: VoiceSnapshot) => void;+ /** The recording hit its cap or the silence limit and is finalizing on its own. */+ onAutoStop?: (reason: "max_duration" | "silence") => void;+ snapshotIntervalMs?: number;+ now?: () => number;+};++type SlotState =+ | { status: "pending" }+ | { status: "done"; text: string }+ | { status: "failed"; error: VoiceError };++type Job = { segment: Segment; wav: Uint8Array | null };++function randomId(): string {+ const bytes = new Uint8Array(8);+ globalThis.crypto.getRandomValues(bytes);+ return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");+}++function wait(ms: number, signal: AbortSignal): Promise<void> {+ return new Promise((resolve) => {+ if (signal.aborted || ms <= 0) return resolve();+ const timer = setTimeout(done, ms);+ function done() {+ clearTimeout(timer);+ signal.removeEventListener("abort", done);+ resolve();+ }+ signal.addEventListener("abort", done, { once: true });+ });+}++/**+ * One dictation: capture → VAD segments → one upload per segment while the+ * user keeps talking (at most `concurrency` at once, retried with backoff) →+ * texts stitched back in recording order.+ *+ * Audio is disposable: each segment's PCM is encoded, uploaded and dropped+ * as soon as its transcript (or final failure) is known; silence is dropped+ * as it arrives; cancel drops everything. Nothing is written anywhere.+ */+export class VoiceSession {+ readonly recordingId: string;+ private readonly options: VoiceSessionOptions;+ private readonly segmenter: Segmenter;+ private readonly meter = new LevelMeter();+ private readonly abort = new AbortController();+ private readonly now: () => number;+ private phaseValue: VoicePhase = "idle";+ private slots: SlotState[] = [];+ private queue: Job[] = [];+ private active = 0;+ private startedAt = 0;+ private lastSpeechAt = 0;+ private lastSnapshotAt = 0;+ private error: VoiceError | null = null;+ private idleWaiters: Array<() => void> = [];+ private capTimer: ReturnType<typeof setInterval> | null = null;+ private result: VoiceResult | null = null;+ private stopping: Promise<VoiceResult> | null = null;++ constructor(options: VoiceSessionOptions) {+ this.options = options;+ this.recordingId = options.recordingId ?? randomId();+ this.segmenter = new Segmenter({ sampleRate: VOICE_SAMPLE_RATE, ...options.segmenter });+ this.now = options.now ?? Date.now;+ }++ get phase(): VoicePhase {+ return this.phaseValue;+ }++ snapshot(): VoiceSnapshot {+ return {+ phase: this.phaseValue,+ levels: this.meter.levels(),+ elapsedMs: this.startedAt ? this.now() - this.startedAt : 0,+ text: this.orderedText(true),+ pending: this.slots.filter((slot) => slot.status === "pending").length,+ failed: this.slots.filter((slot) => slot.status === "failed").length,+ error: this.error,+ };+ }++ async start(): Promise<void> {+ if (this.phaseValue !== "idle") throw new Error("A voice session starts once");+ if (this.options.breaker?.isOpen()) {+ this.fail(new VoiceError("breaker_open", "Voice paused after repeated errors. Try again in a moment."));+ throw this.error;+ }+ this.setPhase("arming");+ try {+ await this.options.source.start(+ (frames) => this.onFrames(frames),+ (error) => {+ if (error) {+ this.abortAll();+ this.fail(error);+ } else if (this.phaseValue === "recording") {+ // The source ran dry (a file input): finalize like a release.+ void this.stop();+ }+ },+ );+ } catch (error) {+ this.options.source.stop();+ this.fail(isVoiceError(error) ? error : new VoiceError("capture_failed", "The microphone could not be opened."));+ throw this.error;+ }+ if (this.phase !== "arming") return;+ this.startedAt = this.now();+ this.lastSpeechAt = this.startedAt;+ this.setPhase("recording");+ this.capTimer = setInterval(() => this.checkCaps(), 250);+ }++ /** Ends capture and waits for the last segment's text. Idempotent. */+ stop(): Promise<VoiceResult> {+ if (this.result) return Promise.resolve(this.result);+ this.stopping ??= this.finalize();+ return this.stopping;+ }++ /** Drops the recording: capture stops, uploads are aborted, no text is returned. */+ cancel(): void {+ if (this.phaseValue === "done" || this.phaseValue === "cancelled") return;+ this.abortAll();+ this.slots = [];+ this.result = { text: "", words: 0, segments: 0, error: null };+ this.setPhase("cancelled");+ this.releaseWaiters();+ }++ private abortAll(): void {+ this.clearCapTimer();+ this.options.source.stop();+ this.abort.abort();+ for (const job of this.queue) job.wav = null;+ this.queue = [];+ }++ private async finalize(): Promise<VoiceResult> {+ if (this.phaseValue === "idle" || this.phaseValue === "arming") this.cancel();+ if (this.phaseValue === "cancelled" || this.phaseValue === "error") {+ this.result ??= { text: "", words: 0, segments: 0, error: this.error };+ return this.result;+ }+ this.clearCapTimer();+ this.options.source.stop();+ this.setPhase("finalizing");+ const last = this.segmenter.flush();+ if (last) this.enqueue(last);+ const deadline = this.options.finalizeTimeoutMs ?? 45_000;+ const timedOut = await Promise.race([+ this.whenIdle().then(() => false),+ wait(deadline, this.abort.signal).then(() => true),+ ]);+ if (this.phase === "cancelled") return this.result ?? { text: "", words: 0, segments: 0, error: null };+ if (timedOut && this.active + this.queue.length > 0) {+ this.abort.abort();+ this.queue = [];+ this.slots = this.slots.map((slot) => slot.status === "pending"+ ? { status: "failed", error: new VoiceError("network", "The transcription service took too long.") }+ : slot);+ }+ const text = this.orderedText(false);+ const error = this.outcomeError(text);+ this.result = { text, words: countWords(text), segments: this.slots.length, error };+ this.error = error;+ this.slots = [];+ this.setPhase("done");+ return this.result;+ }++ private outcomeError(text: string): VoiceError | null {+ const failed = this.slots.find((slot): slot is Extract<SlotState, { status: "failed" }> => slot.status === "failed");+ if (failed) return failed.error;+ if (text) return null;+ if (this.slots.length === 0 && this.segmenter.maxRms < (this.options.noSignalRms ?? 0.003)) {+ return new VoiceError("no_signal", "No audio came from the microphone. Check the input device in Settings → Voice.");+ }+ return new VoiceError("no_speech", "No speech was detected.");+ }++ private onFrames(frames: Int16Array): void {+ // A source may deliver its first frames before start() has resolved.+ if (this.phaseValue !== "recording" && this.phaseValue !== "arming") return;+ const threshold = this.segmenter.threshold;+ const closed = this.segmenter.push(frames, (value) => {+ this.meter.push(value);+ if (value > threshold) this.lastSpeechAt = this.now();+ });+ for (const segment of closed) this.enqueue(segment);+ const at = this.now();+ if (closed.length > 0 || at - this.lastSnapshotAt >= (this.options.snapshotIntervalMs ?? 50)) this.emit();+ }++ private checkCaps(): void {+ if (this.phaseValue !== "recording") return;+ const at = this.now();+ if (at - this.startedAt >= (this.options.maxRecordingMs ?? 120_000)) {+ this.options.onAutoStop?.("max_duration");+ void this.stop();+ return;+ }+ const silence = this.options.silenceAutoStopMs;+ if (silence != null && at - this.lastSpeechAt >= silence) {+ this.options.onAutoStop?.("silence");+ void this.stop();+ }+ }++ private enqueue(segment: Segment): void {+ this.slots[segment.index] = { status: "pending" };+ this.queue.push({ segment, wav: null });+ this.pump();+ this.emit();+ }++ private pump(): void {+ const limit = Math.max(1, this.options.concurrency ?? 2);+ while (this.active < limit && this.queue.length > 0 && !this.abort.signal.aborted) {+ const job = this.queue.shift()!;+ this.active += 1;+ void this.run(job).finally(() => {+ this.active -= 1;+ this.pump();+ if (this.active === 0 && this.queue.length === 0) this.releaseWaiters();+ });+ }+ }++ private async run(job: Job): Promise<void> {+ const { segment } = job;+ job.wav = encodeWav(segment.samples);+ // The PCM is not needed once it is encoded.+ const segmentMeta = { peakRms: segment.peakRms, threshold: segment.threshold, speechMs: segment.speechMs };+ const durationMs = segment.endMs - segment.startMs;+ const delays = this.options.retryDelaysMs ?? [300, 1_200];+ const keyterms = this.options.keyterms ?? [];+ try {+ for (let attempt = 0; ; attempt += 1) {+ if (this.abort.signal.aborted || !job.wav) return;+ try {+ const { text } = await this.options.transcriber.transcribe({+ wav: job.wav,+ segmentIndex: segment.index,+ recordingId: this.recordingId,+ language: this.options.language ?? null,+ keyterms,+ durationMs,+ }, this.abort.signal);+ if (this.abort.signal.aborted) return;+ this.options.breaker?.recordSuccess();+ const cleaned = isLikelyHallucination(text, segmentMeta) ? "" : applyKeyterms(text.trim(), keyterms);+ this.slots[segment.index] = { status: "done", text: cleaned };+ return;+ } catch (error) {+ if (this.abort.signal.aborted) return;+ const voiceError = isVoiceError(error)+ ? error+ : new VoiceError("network", "The transcription service could not be reached.", { retryable: true });+ if (voiceError.retryable && attempt < delays.length) {+ const pause = Math.min(5_000, Math.max(delays[attempt]!, voiceError.retryAfterMs ?? 0));+ await wait(pause, this.abort.signal);+ continue;+ }+ if (voiceError.kind === "network") this.options.breaker?.recordFailure();+ this.slots[segment.index] = { status: "failed", error: voiceError };+ return;+ }+ }+ } finally {+ // Disposable audio: the segment's bytes go as soon as its outcome is known.+ job.wav = null;+ segment.samples = new Int16Array(0);+ this.emit();+ }+ }++ /** Finished texts in order; with `stopAtPending`, up to the first segment still in flight. */+ private orderedText(stopAtPending: boolean): string {+ const parts: string[] = [];+ for (const slot of this.slots) {+ if (!slot) continue;+ if (slot.status === "pending") {+ if (stopAtPending) break;+ continue;+ }+ if (slot.status === "done") parts.push(slot.text);+ }+ return stitch(parts);+ }++ private whenIdle(): Promise<void> {+ if (this.active === 0 && this.queue.length === 0) return Promise.resolve();+ return new Promise((resolve) => this.idleWaiters.push(resolve));+ }++ private releaseWaiters(): void {+ const waiters = this.idleWaiters;+ this.idleWaiters = [];+ for (const resolve of waiters) resolve();+ }++ private clearCapTimer(): void {+ if (this.capTimer) clearInterval(this.capTimer);+ this.capTimer = null;+ }++ private fail(error: VoiceError | null): void {+ this.error = error;+ this.clearCapTimer();+ this.slots = [];+ this.setPhase("error");+ this.releaseWaiters();+ }++ private setPhase(phase: VoicePhase): void {+ this.phaseValue = phase;+ this.emit();+ }++ private emit(): void {+ this.lastSnapshotAt = this.now();+ try {+ this.options.onChange?.(this.snapshot());+ } catch {+ // A listener never breaks the recording.+ }+ }+}diff --git a/apps/app/src/app/lib/voice-core/text.ts b/apps/app/src/app/lib/voice-core/text.tsnew file mode 100644index 00000000..61f16a1e--- /dev/null+++ b/apps/app/src/app/lib/voice-core/text.ts@@ -0,0 +1,81 @@+/** Han, Hiragana, Katakana, Hangul, and CJK punctuation: written without spaces between words. */+const CJK = /[ -〿-ヿ㐀-䶿一-鿿가-豈--]/;+const CJK_GLOBAL = new RegExp(CJK.source, "gu");+/** Punctuation that attaches to the word before it. */+const CLOSING = /^[.,!?;:%)\]}'"’”…、。,!?:;)」』]/u;+const OPENING = /[([{'"‘“「『]$/u;++export function isCjk(char: string): boolean {+ return CJK.test(char);+}++/** Words the way a person counts them: whitespace-separated, and every CJK character on its own. */+export function countWords(text: string): number {+ const cjk = text.match(CJK_GLOBAL)?.length ?? 0;+ const rest = text.replace(CJK_GLOBAL, " ").trim();+ return cjk + (rest ? rest.split(/\s+/).length : 0);+}++/** Whether a space belongs between `left` and `right` (the text on either side of a join). */+function needsSpace(left: string, right: string): boolean {+ if (!left || !right) return false;+ const last = left.slice(-1);+ const first = right.slice(0, 1);+ if (/\s/.test(last) || /\s/.test(first)) return false;+ if (CLOSING.test(right) || OPENING.test(left)) return false;+ if (isCjk(last) && isCjk(first)) return false;+ return true;+}++/** Segment texts in recording order, joined the way they were spoken. */+export function stitch(parts: readonly string[]): string {+ let out = "";+ for (const raw of parts) {+ const part = raw.trim();+ if (!part) continue;+ out += needsSpace(out, part) ? ` ${part}` : part;+ }+ return out;+}++/**+ * `text` as it should be inserted between `before` and `after` (the draft+ * around the cursor): a separating space on each side where a word would+ * otherwise run into its neighbour.+ */+export function padInsertion(before: string, text: string, after: string): string {+ const body = text.trim();+ if (!body) return "";+ return `${needsSpace(before, body) ? " " : ""}${body}${needsSpace(body, after) ? " " : ""}`;+}++/**+ * Whisper-style models answer silence and breath with stock phrases. A+ * segment whose whole text is one of these, from quiet or very short audio,+ * is dropped.+ */+const HALLUCINATIONS = new Set([+ "thank you",+ "thank you very much",+ "thanks",+ "thanks for watching",+ "thank you for watching",+ "please subscribe",+ "you",+ "bye",+ "bye bye",+ "okay",+ "so",+ "uh",+ "um",+ "hmm",+ "music",+ "applause",+]);++export function isLikelyHallucination(text: string, audio: { peakRms: number; threshold: number; speechMs: number }): boolean {+ const normalized = text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").replace(/\s+/g, " ").trim();+ if (!normalized) return true;+ if (!HALLUCINATIONS.has(normalized)) return false;+ return audio.speechMs < 600 || audio.peakRms < audio.threshold * 2;+}diff --git a/apps/app/src/app/lib/voice-core/transcriber.ts b/apps/app/src/app/lib/voice-core/transcriber.tsnew file mode 100644index 00000000..afed30ff--- /dev/null+++ b/apps/app/src/app/lib/voice-core/transcriber.ts@@ -0,0 +1,116 @@+import { VoiceError, type TranscribeRequest, type TranscribeResult, type Transcriber } from "./types";++type FetchLike = (input: string, init: RequestInit) => Promise<Response>;++export type RemoteTranscriberOptions = {+ /** POST target: the backend's /omnirush/v1/audio/transcriptions, or a local broker route in front of it. */+ url: string;+ /** Called per request, so a rotated bearer is picked up. */+ headers?: () => Record<string, string> | Promise<Record<string, string>>;+ fetch?: FetchLike;+ /** Per-request deadline. */+ timeoutMs?: number;+};++function retryAfterMs(response: Response): number | null {+ const header = response.headers.get("retry-after");+ if (!header) return null;+ const seconds = Number(header);+ if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000);+ const date = Date.parse(header);+ return Number.isFinite(date) ? Math.max(0, date - Date.now()) : null;+}++/** The error code of a JSON error body: `{"detail":"x"}`, `{"error":"x"}`, `{"error":{"code":"x"}}` or `{"code":"x"}`. */+export function errorCode(body: unknown): string | null {+ if (!body || typeof body !== "object") return null;+ const record = body as Record<string, unknown>;+ for (const value of [record.detail, record.error, record.code]) {+ if (typeof value === "string" && value.trim()) return value.trim();+ if (value && typeof value === "object") {+ const code = (value as Record<string, unknown>).code;+ if (typeof code === "string" && code.trim()) return code.trim();+ }+ }+ return null;+}++/** Maps a transcription service answer to the error the user reads. */+export function voiceErrorFromResponse(status: number, code: string | null, retryAfter: number | null): VoiceError {+ if (status === 401 || code === "omnirush_account_required") {+ return new VoiceError("signed_out", "Sign in to omnirush.ai to use voice.", { status });+ }+ if (code === "voice_unavailable" || code === "voice_disabled" || status === 404) {+ return new VoiceError("unavailable", "Voice input is not available on this account yet.", { status });+ }+ if (status === 422 || code === "audio_unreadable") {+ return new VoiceError("unreadable", "The recording could not be read. Try again.", { status });+ }+ if (status === 400 || status === 415) {+ return new VoiceError("unreadable", "The recording was not in a format the transcription service accepts.", { status });+ }+ if (status === 413 || code === "audio_too_long" || code === "audio_too_large") {+ return new VoiceError("too_large", "That recording is too long to transcribe.", { status });+ }+ if (status === 429) {+ const budget = code === "voice_daily_budget_exhausted" || code === "daily_grant_exhausted";+ return new VoiceError(+ "rate_limited",+ code === "daily_grant_exhausted"+ ? "You have used today's model allowance, which voice falls back on. It refills at 00:00 UTC."+ : budget ? "You have used today's voice allowance. It refills at 00:00 UTC." : "Voice is busy. Try again in a moment.",+ { status, retryable: !budget, retryAfterMs: retryAfter },+ );+ }+ return new VoiceError("network", "The transcription service did not answer. Try again.", {+ status,+ retryable: status >= 500 || status === 408 || status === 0,+ retryAfterMs: retryAfter,+ });+}++/**+ * A Transcriber over HTTP: one multipart POST per segment (`file`,+ * `language`, `prompt_terms`, `segment_index`, `recording_id`), answered+ * with `{"text": "..."}`. The audio lives only in the request body.+ */+export class RemoteTranscriber implements Transcriber {+ constructor(private readonly options: RemoteTranscriberOptions) {}++ async transcribe(request: TranscribeRequest, signal: AbortSignal): Promise<TranscribeResult> {+ const form = new FormData();+ const wav = request.wav;+ form.append("file", new Blob([wav.buffer.slice(wav.byteOffset, wav.byteOffset + wav.byteLength) as ArrayBuffer], { type: "audio/wav" }), `segment-${request.segmentIndex}.wav`);+ if (request.language) form.append("language", request.language);+ if (request.keyterms.length > 0) form.append("prompt_terms", request.keyterms.join(", "));+ form.append("segment_index", String(request.segmentIndex));+ form.append("recording_id", request.recordingId);+ const fetcher = this.options.fetch ?? ((input: string, init: RequestInit) => fetch(input, init));+ const timeout = AbortSignal.timeout(this.options.timeoutMs ?? 30_000);+ let response: Response;+ try {+ response = await fetcher(this.options.url, {+ method: "POST",+ headers: { Accept: "application/json", ...(await this.options.headers?.()) },+ body: form,+ signal: AbortSignal.any([signal, timeout]),+ });+ } catch (error) {+ if (signal.aborted) throw error;+ throw new VoiceError("network", "The transcription service could not be reached.", { retryable: true, status: 0 });+ }+ const text = await response.text().catch(() => "");+ let body: unknown = null;+ try {+ body = text ? JSON.parse(text) : null;+ } catch {+ body = null;+ }+ if (!response.ok) throw voiceErrorFromResponse(response.status, errorCode(body), retryAfterMs(response));+ const value = body && typeof body === "object" ? (body as Record<string, unknown>).text : null;+ if (typeof value !== "string") {+ throw new VoiceError("network", "The transcription service sent an unexpected answer.", { retryable: true, status: response.status });+ }+ return { text: value };+ }+}diff --git a/apps/app/src/app/lib/voice-core/types.ts b/apps/app/src/app/lib/voice-core/types.tsnew file mode 100644index 00000000..505766ae--- /dev/null+++ b/apps/app/src/app/lib/voice-core/types.ts@@ -0,0 +1,102 @@+/** 16 kHz mono signed 16-bit PCM: the only format voice-core produces and uploads. */+export const VOICE_SAMPLE_RATE = 16_000;++export type VoiceErrorKind =+ /** The microphone delivered only silence (RMS never rose above the no-signal floor). */+ | "no_signal"+ /** There was sound, but nothing that transcribed to words. */+ | "no_speech"+ /** The transcription service could not be reached, or kept failing. */+ | "network"+ /** Voice is switched off for this account or server (503 voice_unavailable). */+ | "unavailable"+ /** The service could not decode the audio (422 audio_unreadable). */+ | "unreadable"+ /** Too many requests or the daily audio budget is spent (429). */+ | "rate_limited"+ /** The account is signed out or its session expired (401). */+ | "signed_out"+ /** The OS or the user refused microphone access. */+ | "permission_denied"+ /** No microphone is connected, or the chosen one is gone. */+ | "no_device"+ /** The microphone exists but could not be opened (in use, driver error). */+ | "capture_failed"+ /** Three failures in a short window: voice pauses for a moment. */+ | "breaker_open"+ /** The recording or a segment was larger than the service accepts. */+ | "too_large";++export class VoiceError extends Error {+ readonly kind: VoiceErrorKind;+ readonly retryable: boolean;+ /** How long the service asked to wait before the next try, when it said. */+ readonly retryAfterMs: number | null;+ readonly status: number | null;++ constructor(kind: VoiceErrorKind, message: string, options: { retryable?: boolean; retryAfterMs?: number | null; status?: number | null } = {}) {+ super(message);+ this.name = "VoiceError";+ this.kind = kind;+ this.retryable = options.retryable ?? false;+ this.retryAfterMs = options.retryAfterMs ?? null;+ this.status = options.status ?? null;+ }+}++export function isVoiceError(value: unknown): value is VoiceError {+ return value instanceof VoiceError;+}++/** One speech segment's upload. */+export type TranscribeRequest = {+ /** A complete RIFF/WAVE file, 16 kHz mono s16le. */+ wav: Uint8Array;+ segmentIndex: number;+ recordingId: string;+ /** BCP-47 hint, or null for auto-detect. */+ language: string | null;+ keyterms: readonly string[];+ durationMs: number;+};++export type TranscribeResult = { text: string };++export interface Transcriber {+ /** Resolves with the segment's text; rejects with a VoiceError (or anything, read as a network failure). */+ transcribe(request: TranscribeRequest, signal: AbortSignal): Promise<TranscribeResult>;+}++/**+ * Where PCM comes from: a microphone, a WAV file (tests), a native recorder+ * (CLI). Frames are 16 kHz mono Int16 of any length.+ */+export interface AudioSource {+ start(onFrames: (frames: Int16Array) => void, onEnd?: (error?: VoiceError) => void): Promise<void>;+ stop(): void;+}++export type VoicePhase = "idle" | "arming" | "recording" | "finalizing" | "done" | "cancelled" | "error";++export type VoiceSnapshot = {+ phase: VoicePhase;+ /** The last 16 input levels, 0..1, oldest first. */+ levels: number[];+ elapsedMs: number;+ /** Text of the finished segments, in order, stopped at the first one still in flight. */+ text: string;+ /** Segments uploaded (or queued) whose text has not arrived yet. */+ pending: number;+ /** Segments that failed for good; their audio is lost, the rest of the text is kept. */+ failed: number;+ error: VoiceError | null;+};++export type VoiceResult = {+ text: string;+ /** Words in `text`, counting CJK characters one by one. */+ words: number;+ segments: number;+ /** Set when nothing usable came back (no signal, no speech, service down) or part of it was lost. */+ error: VoiceError | null;+};diff --git a/apps/app/src/app/lib/voice-core/wav.ts b/apps/app/src/app/lib/voice-core/wav.tsnew file mode 100644index 00000000..777d313c--- /dev/null+++ b/apps/app/src/app/lib/voice-core/wav.ts@@ -0,0 +1,95 @@+import { VOICE_SAMPLE_RATE } from "./types";++/** A RIFF/WAVE file holding `samples` as 16-bit PCM, mono. */+export function encodeWav(samples: Int16Array, sampleRate = VOICE_SAMPLE_RATE): Uint8Array {+ const dataBytes = samples.length * 2;+ const bytes = new Uint8Array(44 + dataBytes);+ const view = new DataView(bytes.buffer);+ const ascii = (offset: number, text: string) => {+ for (let index = 0; index < text.length; index += 1) view.setUint8(offset + index, text.charCodeAt(index));+ };+ ascii(0, "RIFF");+ view.setUint32(4, 36 + dataBytes, true);+ ascii(8, "WAVE");+ ascii(12, "fmt ");+ view.setUint32(16, 16, true);+ view.setUint16(20, 1, true); // PCM+ view.setUint16(22, 1, true); // mono+ view.setUint32(24, sampleRate, true);+ view.setUint32(28, sampleRate * 2, true);+ view.setUint16(32, 2, true);+ view.setUint16(34, 16, true);+ ascii(36, "data");+ view.setUint32(40, dataBytes, true);+ for (let index = 0; index < samples.length; index += 1) view.setInt16(44 + index * 2, samples[index]!, true);+ return bytes;+}++export type DecodedWav = {+ sampleRate: number;+ /** Mono, -1..1 (channels averaged). */+ samples: Float32Array;+};++/**+ * Reads a PCM WAV file: 8/16/24/32-bit integer or 32-bit float, any channel+ * count and sample rate. Used by the file input override, never on mic audio.+ */+export function decodeWav(input: Uint8Array): DecodedWav {+ const view = new DataView(input.buffer, input.byteOffset, input.byteLength);+ const tag = (offset: number) => String.fromCharCode(...input.subarray(offset, offset + 4));+ if (input.byteLength < 12 || tag(0) !== "RIFF" || tag(8) !== "WAVE") throw new Error("Not a WAV file");+ let format = 0;+ let channels = 0;+ let sampleRate = 0;+ let bits = 0;+ let offset = 12;+ while (offset + 8 <= input.byteLength) {+ const id = tag(offset);+ const size = view.getUint32(offset + 4, true);+ const body = offset + 8;+ if (id === "fmt ") {+ format = view.getUint16(body, true);+ channels = view.getUint16(body + 2, true);+ sampleRate = view.getUint32(body + 4, true);+ bits = view.getUint16(body + 14, true);+ if (format === 0xfffe && size >= 26) format = view.getUint16(body + 24, true);+ } else if (id === "data") {+ if (!channels || !sampleRate || !bits) throw new Error("WAV data before its format");+ const end = Math.min(input.byteLength, body + size);+ const width = bits / 8;+ const frames = Math.floor((end - body) / (width * channels));+ const samples = new Float32Array(frames);+ for (let frame = 0; frame < frames; frame += 1) {+ let sum = 0;+ for (let channel = 0; channel < channels; channel += 1) {+ const at = body + (frame * channels + channel) * width;+ sum += readSample(view, at, format, bits);+ }+ samples[frame] = sum / channels;+ }+ return { sampleRate, samples };+ }+ offset = body + size + (size % 2);+ }+ throw new Error("WAV file has no data");+}++function readSample(view: DataView, at: number, format: number, bits: number): number {+ if (format === 3 && bits === 32) return view.getFloat32(at, true);+ if (format !== 1) throw new Error(`Unsupported WAV format ${format}`);+ switch (bits) {+ case 8:+ return (view.getUint8(at) - 128) / 128;+ case 16:+ return view.getInt16(at, true) / 32768;+ case 24: {+ const value = view.getUint8(at) | (view.getUint8(at + 1) << 8) | (view.getInt8(at + 2) << 16);+ return value / 8388608;+ }+ case 32:+ return view.getInt32(at, true) / 2147483648;+ default:+ throw new Error(`Unsupported WAV bit depth ${bits}`);+ }+}diff --git a/apps/app/tests/voice-core.test.ts b/apps/app/tests/voice-core.test.tsnew file mode 100644index 00000000..5230b7c1--- /dev/null+++ b/apps/app/tests/voice-core.test.ts@@ -0,0 +1,447 @@+import { describe, expect, test } from "bun:test";++import {+ CircuitBreaker,+ RemoteTranscriber,+ Segmenter,+ VoiceError,+ VoiceSession,+ WavFileSource,+ applyKeyterms,+ buildKeyterms,+ countWords,+ decodeWav,+ encodeWav,+ padInsertion,+ resampleTo16k,+ stitch,+ voiceInputFileFromEnv,+ type TranscribeRequest,+ type Transcriber,+ type VoiceResult,+ type VoiceSnapshot,+} from "../src/app/lib/voice-core";++const RATE = 16_000;++/** Speech stand-in: a voiced tone with a little vibrato at `amplitude`; silence: faint noise. */+function tone(ms: number, amplitude = 0.3): Int16Array {+ const out = new Int16Array(Math.round((RATE * ms) / 1000));+ for (let i = 0; i < out.length; i += 1) {+ const t = i / RATE;+ out[i] = Math.round(amplitude * 32767 * Math.sin(2 * Math.PI * (180 + 20 * Math.sin(2 * Math.PI * 5 * t)) * t));+ }+ return out;+}++function quiet(ms: number, amplitude = 0.001): Int16Array {+ const out = new Int16Array(Math.round((RATE * ms) / 1000));+ let seed = 7;+ for (let i = 0; i < out.length; i += 1) {+ seed = (seed * 1103515245 + 12345) & 0x7fffffff;+ out[i] = Math.round(((seed / 0x7fffffff) * 2 - 1) * amplitude * 32767);+ }+ return out;+}++function concat(...parts: Int16Array[]): Int16Array {+ const out = new Int16Array(parts.reduce((n, p) => n + p.length, 0));+ let at = 0;+ for (const part of parts) {+ out.set(part, at);+ at += part.length;+ }+ return out;+}++function segmentsOf(pcm: Int16Array, options = {}) {+ const segmenter = new Segmenter(options);+ const out = [];+ // Feed in odd-sized chunks, like an audio callback would.+ for (let at = 0; at < pcm.length; at += 777) out.push(...segmenter.push(pcm.slice(at, at + 777)));+ const last = segmenter.flush();+ if (last) out.push(last);+ return out;+}++describe("segmenter", () => {+ test("silence produces no segment at all", () => {+ expect(segmentsOf(quiet(5_000))).toHaveLength(0);+ expect(segmentsOf(new Int16Array(RATE * 3))).toHaveLength(0);+ });++ test("speech is cut at pauses of 600 ms or more", () => {+ const segments = segmentsOf(concat(quiet(300), tone(1_500), quiet(800), tone(1_400), quiet(900), tone(1_200), quiet(300)));+ expect(segments.map((s) => s.index)).toEqual([0, 1, 2]);+ for (const segment of segments) {+ expect(segment.speechMs).toBeGreaterThan(1_000);+ // Lead-in kept, trailing silence trimmed to the tail.+ expect(segment.samples.length / RATE).toBeLessThan(1.5 + 0.2 + 0.3);+ }+ expect(segments[1]!.startMs).toBeGreaterThan(segments[0]!.endMs);+ });++ test("a short phrase is held open across a normal pause so it is not sent alone", () => {+ const segments = segmentsOf(concat(tone(400), quiet(700), tone(1_500), quiet(900)));+ expect(segments).toHaveLength(1);+ expect(segments[0]!.speechMs).toBeGreaterThan(1_800);+ });++ test("a long pause ends even a short phrase", () => {+ expect(segmentsOf(concat(tone(400), quiet(1_700), tone(1_500), quiet(900)))).toHaveLength(2);+ });++ test("continuous speech is cut at 12 s", () => {+ const segments = segmentsOf(tone(15_000));+ expect(segments).toHaveLength(2);+ expect(segments[0]!.samples.length).toBe(12 * RATE);+ });++ test("a click is dropped as noise", () => {+ expect(segmentsOf(concat(quiet(500), tone(60), quiet(2_000)))).toHaveLength(0);+ });++ test("the noise floor adapts: speech over steady room noise still segments, the noise alone does not", () => {+ expect(segmentsOf(quiet(4_000, 0.008))).toHaveLength(0);+ const noisy = concat(quiet(2_000, 0.008), tone(1_500, 0.2), quiet(1_000, 0.008));+ expect(segmentsOf(noisy)).toHaveLength(1);+ });+});++describe("wav", () => {+ test("encode then decode round-trips 16 kHz mono PCM", () => {+ const pcm = tone(250);+ const wav = encodeWav(pcm);+ expect(new TextDecoder().decode(wav.subarray(0, 4))).toBe("RIFF");+ expect(wav.length).toBe(44 + pcm.length * 2);+ const decoded = decodeWav(wav);+ expect(decoded.sampleRate).toBe(RATE);+ expect(decoded.samples.length).toBe(pcm.length);+ expect(Math.round(decoded.samples[100]! * 32768)).toBe(pcm[100]!);+ });++ test("48 kHz float stereo resamples to 16 kHz mono", () => {+ const frames = 4_800;+ const bytes = new Uint8Array(44 + frames * 2 * 4);+ const view = new DataView(bytes.buffer);+ const put = (at: number, s: string) => [...s].forEach((c, i) => view.setUint8(at + i, c.charCodeAt(0)));+ put(0, "RIFF");+ view.setUint32(4, bytes.length - 8, true);+ put(8, "WAVE");+ put(12, "fmt ");+ view.setUint32(16, 16, true);+ view.setUint16(20, 3, true);+ view.setUint16(22, 2, true);+ view.setUint32(24, 48_000, true);+ view.setUint32(28, 48_000 * 8, true);+ view.setUint16(32, 8, true);+ view.setUint16(34, 32, true);+ put(36, "data");+ view.setUint32(40, frames * 8, true);+ for (let i = 0; i < frames; i += 1) {+ view.setFloat32(44 + i * 8, 0.5, true);+ view.setFloat32(48 + i * 8, 0.1, true);+ }+ const decoded = decodeWav(bytes);+ expect(decoded.sampleRate).toBe(48_000);+ expect(decoded.samples[10]).toBeCloseTo(0.3, 5);+ const pcm = resampleTo16k(decoded.samples, decoded.sampleRate);+ expect(pcm.length).toBe(1_600);+ expect(pcm[5]! / 32767).toBeCloseTo(0.3, 3);+ });++ test("the file input hook is read from the environment", () => {+ expect(voiceInputFileFromEnv({ OMNIRUSH_VOICE_INPUT_FILE: " /tmp/clip.wav " })).toBe("/tmp/clip.wav");+ expect(voiceInputFileFromEnv({})).toBeNull();+ });+});++describe("text", () => {+ test("stitch joins segment texts in order with the right spacing", () => {+ expect(stitch(["And so, my fellow Americans,", "ask not", " ", "what your country can do."])).toBe(+ "And so, my fellow Americans, ask not what your country can do.",+ );+ expect(stitch(["Hello", ", world", "!"])).toBe("Hello, world!");+ expect(stitch(["東京に", "行きます"])).toBe("東京に行きます");+ });++ test("insertion at the cursor gets separating spaces only where needed", () => {+ expect(padInsertion("Fix the", "parser bug", "please")).toBe(" parser bug ");+ expect(padInsertion("Fix the ", "parser bug", " please")).toBe("parser bug");+ expect(padInsertion("", "hello", "")).toBe("hello");+ expect(padInsertion("(", "note", ")")).toBe("note");+ expect(padInsertion("日本", "語", "")).toBe("語");+ });++ test("words are counted with CJK characters one by one", () => {+ expect(countWords("refactor the OAuth middleware")).toBe(4);+ expect(countWords("東京に行きます")).toBe(7);+ expect(countWords(" ")).toBe(0);+ });+});++describe("keyterms", () => {+ test("built from the repository, branch and files, capped at 50 terms / 1024 chars", () => {+ const terms = buildKeyterms({ repo: "omnirush-gui", branch: "feat/voice-mode", files: ["src/useVoiceDictation.ts", "a/b/segmenter.ts"] });+ expect(terms).toContain("omnirush-gui");+ expect(terms).toContain("voice");+ expect(terms).toContain("useVoiceDictation.ts");+ expect(terms).toContain("Dictation");+ expect(terms.filter((t) => t.toLowerCase() === "omnirush")).toHaveLength(1);+ const many = buildKeyterms({ files: Array.from({ length: 200 }, (_, i) => `file${"x".repeat(40)}${i}.ts`) });+ expect(many.length).toBeLessThanOrEqual(50);+ expect(many.join(" ").length).toBeLessThanOrEqual(1024);+ });++ test("post-correction fixes spoken dev terms conservatively", () => {+ expect(applyKeyterms("refactor the oh auth middleware in auth dot ts")).toBe("refactor the OAuth middleware in auth.ts");+ expect(applyKeyterms("push it to git hub and open local host")).toBe("push it to GitHub and open localhost");+ expect(applyKeyterms("open the voice dictation hook", ["VoiceDictation"])).toBe("open the VoiceDictation hook");+ // Plain words and short terms are never rewritten.+ expect(applyKeyterms("the jason file", ["gRPC", "api"])).toBe("the jason file");+ });+});++type Call = { request: TranscribeRequest; signal: AbortSignal };++function fakeTranscriber(answer: (request: TranscribeRequest, attempt: number) => Promise<string> | string): Transcriber & { calls: Call[]; maxActive: number } {+ let active = 0;+ const attempts = new Map<number, number>();+ const self = {+ calls: [] as Call[],+ maxActive: 0,+ async transcribe(request: TranscribeRequest, signal: AbortSignal) {+ self.calls.push({ request, signal });+ active += 1;+ self.maxActive = Math.max(self.maxActive, active);+ const attempt = attempts.get(request.segmentIndex) ?? 0;+ attempts.set(request.segmentIndex, attempt + 1);+ try {+ return { text: await answer(request, attempt) };+ } finally {+ active -= 1;+ }+ },+ };+ return self;+}++async function dictate(pcm: Int16Array, transcriber: Transcriber, options: Partial<ConstructorParameters<typeof VoiceSession>[0]> = {}) {+ const snapshots: VoiceSnapshot[] = [];+ let resolveDone!: (result: VoiceResult) => void;+ const done = new Promise<VoiceResult>((resolve) => {+ resolveDone = resolve;+ });+ const session: VoiceSession = new VoiceSession({+ source: new WavFileSource(encodeWav(pcm), { realtime: false }),+ transcriber,+ retryDelaysMs: [5, 10],+ ...options,+ onChange: (snapshot) => {+ snapshots.push(snapshot);+ if (snapshot.phase === "done" || snapshot.phase === "cancelled" || snapshot.phase === "error") void session.stop().then(resolveDone);+ },+ });+ await session.start();+ return { session, snapshots, result: await done };+}++const THREE_PHRASES = concat(tone(1_500), quiet(800), tone(1_500), quiet(800), tone(1_500), quiet(400));+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++describe("voice session", () => {+ test("uploads each segment as a valid 16 kHz WAV and stitches the texts in recording order even when answers arrive out of order", async () => {+ const words = ["refactor the", "OAuth middleware", "in auth dot ts"];+ const transcriber = fakeTranscriber(async (request) => {+ await sleep(40 - request.segmentIndex * 15);+ return words[request.segmentIndex]!;+ });+ const { result, snapshots } = await dictate(THREE_PHRASES, transcriber);+ expect(transcriber.calls).toHaveLength(3);+ expect(transcriber.maxActive).toBeLessThanOrEqual(2);+ for (const { request } of transcriber.calls) {+ const decoded = decodeWav(request.wav);+ expect(decoded.sampleRate).toBe(RATE);+ expect(request.recordingId).toMatch(/^[0-9a-f]{16}$/);+ }+ expect(result.text).toBe("refactor the OAuth middleware in auth.ts");+ expect(result.words).toBe(6);+ expect(result.error).toBeNull();+ // Partial text only ever grows from the start (never shows segment 2 before segment 1).+ const partials = snapshots.map((s) => s.text).filter(Boolean);+ for (const partial of partials) expect(result.text.startsWith(partial.replace(" dot ts", ""))).toBe(true);+ });++ test("a silent recording uploads nothing and reports no signal", async () => {+ const transcriber = fakeTranscriber(() => "Thank you.");+ const { result } = await dictate(new Int16Array(RATE * 3), transcriber);+ expect(transcriber.calls).toHaveLength(0);+ expect(result.text).toBe("");+ expect(result.error?.kind).toBe("no_signal");+ });++ test("room noise without speech uploads nothing and reports no speech", async () => {+ const transcriber = fakeTranscriber(() => "x");+ const { result } = await dictate(quiet(3_000, 0.006), transcriber);+ expect(transcriber.calls).toHaveLength(0);+ expect(result.error?.kind).toBe("no_speech");+ });++ test("a stock hallucination from short audio is dropped", async () => {+ const transcriber = fakeTranscriber(() => "Thank you.");+ const { result } = await dictate(concat(tone(400), quiet(2_000)), transcriber);+ expect(transcriber.calls).toHaveLength(1);+ expect(result.text).toBe("");+ expect(result.error?.kind).toBe("no_speech");+ });++ test("transient failures are retried with backoff", async () => {+ const transcriber = fakeTranscriber((request, attempt) => {+ if (attempt < 2) throw new VoiceError("network", "down", { retryable: true, status: 502 });+ return `part ${request.segmentIndex}`;+ });+ const { result } = await dictate(THREE_PHRASES, transcriber);+ expect(transcriber.calls).toHaveLength(9);+ expect(result.text).toBe("part 0 part 1 part 2");+ expect(result.error).toBeNull();+ });++ test("a segment that keeps failing is lost but the rest of the text is kept", async () => {+ const transcriber = fakeTranscriber((request) => {+ if (request.segmentIndex === 1) throw new VoiceError("unreadable", "bad audio", { status: 422 });+ return `part ${request.segmentIndex}`;+ });+ const { result } = await dictate(THREE_PHRASES, transcriber);+ expect(transcriber.calls).toHaveLength(3);+ expect(result.text).toBe("part 0 part 2");+ expect(result.error?.kind).toBe("unreadable");+ });++ test("voice unavailable is not retried", async () => {+ const transcriber = fakeTranscriber(() => {+ throw new VoiceError("unavailable", "off", { status: 503 });+ });+ const { result } = await dictate(concat(tone(1_500), quiet(900)), transcriber);+ expect(transcriber.calls).toHaveLength(1);+ expect(result.error?.kind).toBe("unavailable");+ });++ test("cancel aborts uploads in flight and returns no text", async () => {+ const transcriber = fakeTranscriber(async () => {+ await sleep(5_000);+ return "late";+ });+ const session = new VoiceSession({+ source: new WavFileSource(encodeWav(concat(tone(1_500), quiet(900), tone(20_000))), { realtime: true, chunkMs: 20 }),+ transcriber,+ });+ await session.start();+ while (transcriber.calls.length === 0) await sleep(20);+ session.cancel();+ expect(session.phase).toBe("cancelled");+ expect(transcriber.calls[0]!.signal.aborted).toBe(true);+ const result = await session.stop();+ expect(result.text).toBe("");+ expect(session.snapshot().pending).toBe(0);+ });++ test("the breaker opens after three failures in ten seconds", async () => {+ let now = 1_000;+ const breaker = new CircuitBreaker(3, 10_000, 30_000, () => now);+ breaker.recordFailure();+ breaker.recordFailure();+ expect(breaker.isOpen()).toBe(false);+ breaker.recordFailure();+ expect(breaker.isOpen()).toBe(true);+ const session = new VoiceSession({ source: new WavFileSource(encodeWav(tone(100))), transcriber: fakeTranscriber(() => ""), breaker });+ await expect(session.start()).rejects.toMatchObject({ kind: "breaker_open" });+ now += 31_000;+ expect(breaker.isOpen()).toBe(false);+ });++ test("tap mode stops by itself after the silence limit", async () => {+ const reasons: string[] = [];+ const transcriber = fakeTranscriber(() => "hello there");+ const session = new VoiceSession({+ source: new WavFileSource(encodeWav(concat(tone(1_200), quiet(3_000))), { realtime: true, chunkMs: 20 }),+ transcriber,+ silenceAutoStopMs: 900,+ onAutoStop: (reason) => reasons.push(reason),+ });+ await session.start();+ const started = Date.now();+ while (session.phase === "recording" && Date.now() - started < 4_000) await sleep(25);+ const result = await session.stop();+ expect(reasons).toEqual(["silence"]);+ expect(Date.now() - started).toBeLessThan(3_000);+ expect(result.text).toBe("hello there");+ });+});++describe("remote transcriber", () => {+ const request: TranscribeRequest = {+ wav: encodeWav(tone(100)),+ segmentIndex: 2,+ recordingId: "abc",+ language: "en",+ keyterms: ["OmniRush", "voice"],+ durationMs: 100,+ };++ test("posts a multipart segment and reads the text", async () => {+ let seen: FormData | null = null;+ let headers: Record<string, string> = {};+ const transcriber = new RemoteTranscriber({+ url: "http://127.0.0.1:1/omnirush/voice/transcribe",+ headers: () => ({ Authorization: "Bearer t" }),+ fetch: async (_url, init) => {+ seen = init.body as FormData;+ headers = init.headers as Record<string, string>;+ return Response.json({ text: "hello", provider: "primary", duration_ms: 100 });+ },+ });+ expect(await transcriber.transcribe(request, new AbortController().signal)).toEqual({ text: "hello" });+ const form = seen as unknown as FormData;+ expect(form.get("segment_index")).toBe("2");+ expect(form.get("recording_id")).toBe("abc");+ expect(form.get("language")).toBe("en");+ expect(form.get("prompt_terms")).toBe("OmniRush, voice");+ const file = form.get("file") as File;+ expect(file.type).toBe("audio/wav");+ expect(file.size).toBe(request.wav.length);+ expect(headers.Authorization).toBe("Bearer t");+ });++ test.each([+ [503, { detail: "voice_unavailable" }, "unavailable", false],+ [503, { error: { code: "transcription_unavailable" } }, "network", true],+ [422, { detail: "audio_unreadable" }, "unreadable", false],+ [429, { detail: "rate_limited" }, "rate_limited", true],+ [429, { detail: "voice_daily_budget_exhausted" }, "rate_limited", false],+ [429, { detail: "voice_concurrency_limited" }, "rate_limited", true],+ [413, { detail: "audio_too_long" }, "too_large", false],+ [415, { detail: "unsupported_audio_format" }, "unreadable", false],+ [400, { detail: "invalid_multipart" }, "unreadable", false],+ [429, { detail: "daily_grant_exhausted" }, "rate_limited", false],+ [503, { detail: "voice_budget_unavailable" }, "network", true],+ [401, { error: "omnirush_account_required" }, "signed_out", false],+ [413, { detail: "audio_too_large" }, "too_large", false],+ ] as const)("HTTP %i %j → %s", async (status, body, kind, retryable) => {+ const transcriber = new RemoteTranscriber({+ url: "http://x",+ fetch: async () => new Response(JSON.stringify(body), { status, headers: { "retry-after": "2" } }),+ });+ const error = await transcriber.transcribe(request, new AbortController().signal).catch((e: unknown) => e);+ expect(error).toBeInstanceOf(VoiceError);+ expect((error as VoiceError).kind).toBe(kind);+ expect((error as VoiceError).retryable).toBe(retryable);+ if (status === 429) expect((error as VoiceError).retryAfterMs).toBe(2_000);+ });++ test("a connection failure is a retryable network error", async () => {+ const transcriber = new RemoteTranscriber({ url: "http://x", fetch: async () => { throw new TypeError("fetch failed"); } });+ const error = await transcriber.transcribe(request, new AbortController().signal).catch((e: unknown) => e);+ expect((error as VoiceError).kind).toBe("network");+ expect((error as VoiceError).retryable).toBe(true);+ });+});