Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
// Browser audio recording → 16 kHz mono WAV → base64, for one-shot memo
// capture (catalog voice ingest). whisper.cpp accepts WAV only, so we decode
// whatever MediaRecorder produced and resample to 16 kHz mono before encoding.
//
// `startMemoRecording` and the analysis-mic helpers below are standalone
// "record a clip, get a WAV" plumbing — NOT wired to the live voice-agent
// socket pipeline (echo gating, VAD, streaming TTS) that owns
// services/voiceClient.js. The WAV-encode/resample helpers (`encodePcmToWav`,
// `blobToWav16k`, `float32ToWav16k`) ARE shared with that module — it imports
// them rather than keeping its own copy.
import { SOCKET_MEDIA_MAX_BYTES } from '../../../server/lib/socketPayloadLimits.js';
import { resumeAudioContext, acquireAudioSession } from './audioContext.js';
export const MAX_VOICE_AUDIO_BYTES = SOCKET_MEDIA_MAX_BYTES;
export const RECORDING_TOO_LONG_MESSAGE = 'Recording too long. Please record a shorter message.';
export const TARGET_SAMPLE_RATE = 16000;
// Pick a MediaRecorder mime the browser supports; Safari lands on mp4, others
// on webm/opus. We re-decode to WAV regardless, so the intermediate codec
// only needs to be recordable + decodable by Web Audio.
export function pickRecordingMimeType() {
const candidates = ['audio/webm;codecs=opus', 'audio/webm', 'audio/ogg;codecs=opus', 'audio/mp4'];
for (const m of candidates) {
if (window.MediaRecorder && window.MediaRecorder.isTypeSupported?.(m)) return m;
}
return 'audio/webm';
}
// Encode a mono Float32 PCM buffer to a 16-bit WAV ArrayBuffer.
export function encodePcmToWav(float32, sampleRate = TARGET_SAMPLE_RATE) {
const n = float32.length;
const buffer = new ArrayBuffer(44 + n * 2);
const view = new DataView(buffer);
const writeStr = (off, s) => { for (let i = 0; i < s.length; i++) view.setUint8(off + i, s.charCodeAt(i)); };
writeStr(0, 'RIFF');
view.setUint32(4, 36 + n * 2, true);
writeStr(8, 'WAVE');
writeStr(12, 'fmt ');
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, 1, true);
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * 2, true);
view.setUint16(32, 2, true);
view.setUint16(34, 16, true);
writeStr(36, 'data');
view.setUint32(40, n * 2, true);
let off = 44;
for (let i = 0; i < n; i++) {
const s = Math.max(-1, Math.min(1, float32[i]));
view.setInt16(off, s < 0 ? s * 0x8000 : s * 0x7fff, true);
off += 2;
}
return buffer;
}
// Decode a recorded blob (any Web-Audio-decodable codec) → 16 kHz mono WAV.
// Returns `{ wav: ArrayBuffer, peak: number }`; peak amplitude surfaces a
// dead/too-quiet mic before we waste a Whisper round-trip on silence.
export async function blobToWav16k(blob) {
const bytes = await blob.arrayBuffer();
const decodeCtx = new (window.AudioContext || window.webkitAudioContext)();
const decoded = await decodeCtx.decodeAudioData(bytes).finally(() => {
decodeCtx.close().catch(() => {});
});
const frames = Math.ceil(decoded.duration * TARGET_SAMPLE_RATE);
const offline = new OfflineAudioContext(1, frames, TARGET_SAMPLE_RATE);
const src = offline.createBufferSource();
src.buffer = decoded;
src.connect(offline.destination);
src.start();
const rendered = await offline.startRendering();
const pcm = rendered.getChannelData(0);
let peak = 0;
for (let i = 0; i < pcm.length; i++) {
const a = Math.abs(pcm[i]);
if (a > peak) peak = a;
}
return { wav: encodePcmToWav(pcm, TARGET_SAMPLE_RATE), peak };
}
// Encode a Float32 PCM buffer captured at `sourceRate` (e.g. an AudioWorklet's
// continuous-mode capture) → 16 kHz mono WAV. Same resample path as
// `blobToWav16k`, but the source is already raw PCM rather than an encoded
// blob. Returns `{ wav: null, peak: 0 }` for an empty buffer — nothing to
// render — so callers can skip a zero-length OfflineAudioContext.
export async function float32ToWav16k(samples, sourceRate) {
if (!samples.length) return { wav: null, peak: 0 };
const frames = Math.ceil(samples.length * TARGET_SAMPLE_RATE / sourceRate);
const offline = new OfflineAudioContext(1, frames, TARGET_SAMPLE_RATE);
const buf = offline.createBuffer(1, samples.length, sourceRate);
buf.getChannelData(0).set(samples);
const src = offline.createBufferSource();
src.buffer = buf;
src.connect(offline.destination);
src.start();
const rendered = await offline.startRendering();
const pcm = rendered.getChannelData(0);
let peak = 0;
for (let i = 0; i < pcm.length; i++) {
const a = Math.abs(pcm[i]);
if (a > peak) peak = a;
}
return { wav: encodePcmToWav(pcm, TARGET_SAMPLE_RATE), peak };
}
// Base64-encode an ArrayBuffer in chunks (avoids the call-stack blowup of
// String.fromCharCode(...bigArray) on multi-second recordings).
export function arrayBufferToBase64(buffer) {
const bytes = new Uint8Array(buffer);
let binary = '';
const CHUNK = 0x8000;
for (let i = 0; i < bytes.length; i += CHUNK) {
binary += String.fromCharCode.apply(null, bytes.subarray(i, i + CHUNK));
}
return btoa(binary);
}
/**
* Tap a live `AnalyserNode` off an existing `MediaStream` — for the live tuner,
* which must read the SAME mic the recorder already opened rather than calling
* getUserMedia a second time (a second stream prompts again / fights the first).
* Returns `{ analyser, context, close }`; `close()` tears down the analyser graph
* and AudioContext but leaves the stream alone (the recorder owns the stream's
* lifetime). Caller MUST call `close()` on stop/unmount (deferred-work cleanup).
*/
export function createStreamAnalyser(stream, { fftSize = 2048 } = {}) {
const context = new (window.AudioContext || window.webkitAudioContext)();
// A context created outside a user gesture (e.g. in a render-driven effect)
// can start `suspended` — or, on iOS, `'interrupted'`; resume so frame reads
// aren't browser-dependent. Fire-and-forget: the analyser is read per frame,
// so a late resume just means the first frames read zeros.
resumeAudioContext(context).catch(() => {});
const source = context.createMediaStreamSource(stream);
const analyser = context.createAnalyser();
analyser.fftSize = fftSize;
source.connect(analyser); // no connection to destination — we only read frames
return {
analyser,
context,
close: () => {
source.disconnect();
analyser.disconnect();
context.close().catch(() => {});
},
};
}
/**
* Start recording from the default microphone. Returns a handle whose
* `stop()` resolves to `{ audioBase64, peak, mimeType, durationMs }` — a
* 16 kHz mono WAV base64 string ready to POST. The handle also exposes the live
* `stream` so an analyser (live tuner) can tap the same mic without opening a
* second one. The caller is responsible for calling `stop()` (or `cancel()` to
* discard). Throws if mic access is denied.
*/
export async function startMemoRecording() {
// Claimed BEFORE getUserMedia — an output-only `playback` session held by a
// play-along elsewhere on the page would refuse the request outright — and
// handed back if the mic is denied, since neither stop() nor cancel() runs
// when this throws. See the audio-session note in audioContext.js.
const releaseSession = acquireAudioSession('play-and-record');
const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
.catch((err) => { releaseSession(); throw err; });
const mimeType = pickRecordingMimeType();
const recorder = new MediaRecorder(stream, { mimeType });
const chunks = [];
const startedAt = Date.now();
recorder.ondataavailable = (e) => { if (e.data && e.data.size > 0) chunks.push(e.data); };
recorder.start();
// Both exit paths (stop and cancel) run this, and the release is idempotent.
const teardown = () => {
stream.getTracks().forEach((t) => t.stop());
releaseSession();
};
return {
stream,
stop: () => new Promise((resolve, reject) => {
recorder.onstop = async () => {
teardown();
try {
const blob = new Blob(chunks, { type: mimeType });
const { wav, peak } = await blobToWav16k(blob);
resolve({
audioBase64: arrayBufferToBase64(wav),
mimeType: 'audio/wav',
peak,
durationMs: Date.now() - startedAt,
});
} catch (err) {
reject(err);
}
};
recorder.stop();
}),
cancel: () => { teardown(); try { recorder.stop(); } catch { /* already stopped */ } },
};
}
/**
* Constraints every PITCH-ANALYSIS mic opens with. The browser's default
* capture chain is tuned for speech intelligibility, not signal analysis: AGC
* rides the level the tuner's clarity gate reads, noise suppression chews
* sustained vowels and soft onsets, and echo cancellation can gate the mic
* outright while a reference melody plays back. Plain booleans, NOT `{ exact }`
* — a browser that can't honor one must still open the mic rather than fail
* closed; `openAnalysisMic` reports back what actually stuck.
*
* Deliberately NOT applied to `startMemoRecording` — that capture feeds Whisper
* transcription, where the speech-tuned chain helps.
*/
export const ANALYSIS_AUDIO_CONSTRAINTS = Object.freeze({
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
});
const PROCESSING_KEYS = ['echoCancellation', 'noiseSuppression', 'autoGainControl'];
/**
* Read back which processing stages the browser ACTUALLY applied to a capture
* stream. Each key is `true` (on despite the constraint), `false` (honored), or
* `null` when the browser doesn't report it — Firefox omits keys from
* `getSettings()`, and unknown must not collapse into "off" or every such
* browser would silently claim clean audio.
*/
export function readAppliedProcessing(stream) {
const track = stream?.getAudioTracks?.()?.[0] ?? stream?.getTracks?.()?.[0];
const settings = track?.getSettings?.();
return Object.fromEntries(PROCESSING_KEYS.map((key) => [
key,
typeof settings?.[key] === 'boolean' ? settings[key] : null,
]));
}
/** True only when a stage is KNOWN to still be on — `null` (unknown) never warns. */
export function hasUnwantedProcessing(processing) {
return PROCESSING_KEYS.some((key) => processing?.[key] === true);
}
/**
* Open a microphone for pitch/signal analysis with the browser's processing
* chain requested off, and report what it actually applied. Returns
* `{ stream, processing }`; rejects exactly like `getUserMedia` does, so call
* sites keep their existing `.catch()` + re-entrancy guards.
*
* Pass `getUserMedia` when the caller already resolved (and null-checked) it —
* reaching through `navigator.mediaDevices` throws synchronously on an insecure
* origin, and the capture hooks must do that check before claiming the audio
* session.
*/
export async function openAnalysisMic({ getUserMedia } = {}) {
const open = getUserMedia
|| navigator.mediaDevices?.getUserMedia?.bind(navigator.mediaDevices);
if (!open) throw new Error('Microphone access requires a secure browser connection');
const stream = await open({ audio: { ...ANALYSIS_AUDIO_CONSTRAINTS } });
return { stream, processing: readAppliedProcessing(stream) };
}