Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* The FaceTime Audio call host — and its second mode, meeting capture.
*
* Both modes share one audio bridge: a browser tab opens BlackHole 16ch
* (what FaceTime, Zoom, or Meet plays INTO) and streams it to the server as
* 16 kHz mono PCM. It has to run in a real browser tab on the Mac because
* device permissions and `setSinkId` need a real browser profile.
*
* - **Call** plays the assistant's reply back out through BlackHole 2ch
* (what FaceTime listens to as its microphone) — the full voice pipeline.
* - **Capture system audio** only transcribes. It never talks back and needs
* no BlackHole 2ch: it whisper-transcribes the meeting continuously and,
* on stop, files the transcript into the daily journal and the Brain
* inbox for review — no AI provider is called along the way.
*
* Everything here fails closed and says why. A bridge that half-works drops
* audio silently, so a missing API, a missing device, an ungranted
* permission, and a second tab each produce their own message rather than a
* spinner that never resolves.
*/
import { useCallback, useEffect, useRef, useState } from 'react';
import { Mic, MicOff, PhoneCall, Radio, Volume2 } from 'lucide-react';
import socket from '../services/socket';
import TabPills from '../components/ui/TabPills';
import useDrawerTab from '../hooks/useDrawerTab.js';
import {
CALL_FRAME_SAMPLES,
CALL_HOST_LOCK,
buildTestTone,
describeDeviceProblem,
downmixToMono,
floatToInt16,
missingCallHostApis,
resampleTo16k,
rmsLevel,
} from '../lib/callAudioBridge';
// A worklet has to be loaded from a URL, and this one is four lines, so it is
// built from source at runtime rather than carried as a separate build entry.
// It does no processing: the pure helpers do the downmix/resample/quantize on
// the main thread, where they can be tested without an audio graph.
const WORKLET_SOURCE = `
class CallTapProcessor extends AudioWorkletProcessor {
process(inputs) {
const channels = inputs[0];
if (channels?.length) this.port.postMessage(channels.map((c) => new Float32Array(c)));
return true;
}
}
registerProcessor('portos-call-tap', CallTapProcessor);
`;
const DEFAULT_INPUT_LABEL = 'BlackHole 16ch';
const DEFAULT_OUTPUT_LABEL = 'BlackHole 2ch';
const MODES = ['call', 'capture'];
const releaseCallAudio = (playing, element) => {
if (!playing?.delete(element)) return;
element.pause?.();
if (element.src) {
URL.revokeObjectURL(element.src);
element.src = '';
}
};
export default function VoiceCallHost() {
// Deep-linkable mode, not local state: `?mode=capture` is shareable and
// survives a reload, same convention as a drawer's active tab.
const [mode, setMode] = useDrawerTab('mode', 'call', MODES);
// Read inside callbacks that only close over `[]`/`[teardown]` deps, so a
// mode switch between mount and the next attach/detach isn't missed.
const modeRef = useRef(mode);
modeRef.current = mode;
const [blocked, setBlocked] = useState(null);
const [attached, setAttached] = useState(false);
const [callState, setCallState] = useState(null);
const [captureState, setCaptureState] = useState(null);
const [level, setLevel] = useState(0);
const [notice, setNotice] = useState(null);
const audio = useRef({ context: null, stream: null, node: null, source: null, playing: new Set(), outputId: null, pending: [] });
const lockRelease = useRef(null);
const startGenRef = useRef(0);
const releaseLock = useCallback(() => {
lockRelease.current?.();
lockRelease.current = null;
}, []);
const teardown = useCallback(() => {
const current = audio.current;
for (const element of current.playing) releaseCallAudio(current.playing, element);
current.node?.port?.close?.();
current.node?.disconnect?.();
current.source?.disconnect?.();
current.stream?.getTracks?.().forEach((track) => track.stop());
current.context?.close?.();
audio.current = { context: null, stream: null, node: null, source: null, playing: new Set(), outputId: current.outputId, pending: [] };
setLevel(0);
}, []);
// Play one WAV into BlackHole 2ch. `setSinkId` is what makes this land in the
// call instead of the room's speakers, so a failure to route is surfaced
// rather than played anyway. Capture mode never receives a voice:call:tts
// event (the server never emits one for a capture session), so this only
// ever runs for a call.
const playToCall = useCallback(async (wav) => {
const element = new Audio();
const blob = new Blob([wav], { type: 'audio/wav' });
element.src = URL.createObjectURL(blob);
audio.current.playing.add(element);
const release = () => releaseCallAudio(audio.current.playing, element);
element.addEventListener('ended', release, { once: true });
element.addEventListener('error', release, { once: true });
try {
if (audio.current.outputId) await element.setSinkId(audio.current.outputId);
if (!audio.current.playing.has(element)) return;
await element.play();
if (!audio.current.playing.has(element)) return;
} catch (error) {
if (audio.current.playing.has(element)) {
setNotice(`Could not play into the call: ${error.message}`);
releaseCallAudio(audio.current.playing, element);
}
}
}, []);
const start = useCallback(async () => {
const generation = startGenRef.current;
setNotice(null);
const missing = missingCallHostApis(window);
if (missing.length) {
setBlocked(`This browser is missing ${missing.join(', ')}. Use Chrome on the Mac running PortOS.`);
return;
}
// The lock is the local half of single-attach; the server refuses a second
// host too, but the lock catches it before any device is opened. One lock
// name for both modes — they read the same physical BlackHole device, so
// a second tab must be refused regardless of which mode it picks.
//
// `locks.request` resolves only when its CALLBACK settles, and holding a
// lock means never settling — so awaiting the request itself would hang
// here forever while holding the lock. The grant decision is surfaced out
// of the callback instead, and the held promise is left open on purpose.
const granted = await new Promise((resolve) => {
navigator.locks.request(CALL_HOST_LOCK, { ifAvailable: true }, (held) => {
if (!held) {
resolve(false);
return null;
}
resolve(true);
return new Promise((release) => { lockRelease.current = release; });
}).catch(() => resolve(false));
});
if (generation !== startGenRef.current) {
if (granted) releaseLock();
return;
}
if (!granted) {
setBlocked('Another tab owns the call host. Close it, or use that tab instead.');
return;
}
const currentMode = modeRef.current;
const devices = await navigator.mediaDevices.enumerateDevices().catch(() => null);
if (generation !== startGenRef.current) {
releaseLock();
return;
}
const problem = describeDeviceProblem(devices, {
inputLabel: DEFAULT_INPUT_LABEL,
// Capture only listens — it never plays a reply back, so it has no
// BlackHole 2ch requirement.
outputLabel: currentMode === 'call' ? DEFAULT_OUTPUT_LABEL : undefined,
});
if (problem) {
setBlocked(problem);
releaseLock();
return;
}
const input = devices.find((device) => device.label === DEFAULT_INPUT_LABEL && device.kind === 'audioinput');
const output = currentMode === 'call'
? devices.find((device) => device.label === DEFAULT_OUTPUT_LABEL && device.kind === 'audiooutput')
: null;
let stream;
try {
stream = await navigator.mediaDevices.getUserMedia({
// Every processing stage is off: this is not a room microphone, it is
// the far end of a phone call, and echo cancellation would chew holes
// in it. Plain booleans, never `{ exact }`, so a browser that cannot
// honour one still opens the device.
audio: {
deviceId: { exact: input.deviceId },
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
},
});
} catch (error) {
if (generation === startGenRef.current) setBlocked(`Could not open ${DEFAULT_INPUT_LABEL}: ${error.message}`);
releaseLock();
return;
}
if (generation !== startGenRef.current) {
stream.getTracks().forEach((track) => track.stop());
releaseLock();
return;
}
let context;
let node;
let source;
try {
context = new AudioContext();
const url = URL.createObjectURL(new Blob([WORKLET_SOURCE], { type: 'application/javascript' }));
try {
await context.audioWorklet.addModule(url);
} finally {
URL.revokeObjectURL(url);
}
if (generation !== startGenRef.current) return;
source = context.createMediaStreamSource(stream);
node = new AudioWorkletNode(context, 'portos-call-tap');
source.connect(node);
} catch (error) {
if (generation === startGenRef.current) setBlocked(`Could not start call audio: ${error.message}`);
return;
} finally {
if (generation !== startGenRef.current || !node) {
node?.port?.close?.();
node?.disconnect?.();
source?.disconnect?.();
stream.getTracks().forEach((track) => track.stop());
context?.close?.();
releaseLock();
}
}
let carry = [];
node.port.onmessage = (event) => {
const mono = downmixToMono(event.data);
setLevel(rmsLevel(mono));
const resampled = resampleTo16k(mono, context.sampleRate);
carry.push(...resampled);
// Ship fixed-size frames so the server's endpointer sees a steady
// resolution regardless of the render quantum this browser chose.
// Both modes ride the same wire event — the server routes it to
// whichever host (call or capture) this socket currently holds.
while (carry.length >= CALL_FRAME_SAMPLES) {
const pcm = floatToInt16(Float32Array.from(carry.slice(0, CALL_FRAME_SAMPLES)));
carry = carry.slice(CALL_FRAME_SAMPLES);
socket.emit('voice:call:audio', { pcm: pcm.buffer });
}
};
audio.current = { context, stream, node, source, playing: new Set(), outputId: output?.deviceId ?? null, pending: [] };
// Two literal emits, not a computed event name — a computed
// `socket.emit(x ? 'a' : 'b')` is invisible to the static
// socket-event inventory scan (`server/lib/socketEventInventory.js`),
// which greps for `socket.emit('literal')`.
if (currentMode === 'call') socket.emit('voice:call:attach');
else socket.emit('voice:capture:start');
setBlocked(null);
}, [releaseLock]);
const stop = useCallback(() => {
startGenRef.current += 1;
if (modeRef.current === 'call') socket.emit('voice:call:detach');
else socket.emit('voice:capture:stop');
teardown();
releaseLock();
setAttached(false);
}, [releaseLock, teardown]);
useEffect(() => {
const onCallState = (snapshot) => {
if (snapshot?.error === 'host-taken') {
setBlocked('Another tab owns the call host.');
setAttached(false);
return;
}
if (snapshot?.error === 'capture-active') {
setBlocked('Meeting capture is running on this tab — stop it before attaching a call.');
setAttached(false);
return;
}
setCallState(snapshot);
setAttached(Boolean(snapshot?.hostAttached));
};
const onCaptureState = (snapshot) => {
if (snapshot?.error === 'host-taken') {
setBlocked('Another tab is already capturing a meeting.');
setAttached(false);
return;
}
if (snapshot?.error === 'call-active') {
setBlocked('A FaceTime call is active on this tab — hang up before starting a capture.');
setAttached(false);
return;
}
setCaptureState(snapshot);
setAttached(Boolean(snapshot?.hostAttached));
};
const onTts = ({ wav }) => { if (wav) playToCall(wav); };
const onTtsCancel = () => {
for (const element of audio.current.playing) {
releaseCallAudio(audio.current.playing, element);
}
};
socket.on('voice:call:state', onCallState);
socket.on('voice:capture:state', onCaptureState);
socket.on('voice:call:tts', onTts);
socket.on('voice:tts:cancel', onTtsCancel);
return () => {
startGenRef.current += 1;
socket.off('voice:call:state', onCallState);
socket.off('voice:capture:state', onCaptureState);
socket.off('voice:call:tts', onTts);
socket.off('voice:tts:cancel', onTtsCancel);
if (modeRef.current === 'call') socket.emit('voice:call:detach');
else socket.emit('voice:capture:stop');
teardown();
releaseLock();
};
}, [playToCall, releaseLock, teardown]);
const playTestTone = async () => {
const context = audio.current.context || new AudioContext();
const tone = buildTestTone(context.sampleRate, 1);
const buffer = context.createBuffer(1, tone.length, context.sampleRate);
buffer.copyToChannel(tone, 0);
const source = context.createBufferSource();
source.buffer = buffer;
const destination = context.createMediaStreamDestination();
source.connect(destination);
const element = new Audio();
element.srcObject = destination.stream;
if (audio.current.outputId) await element.setSinkId(audio.current.outputId).catch(() => {});
await element.play().catch((error) => setNotice(`Test tone failed: ${error.message}`));
source.start();
};
const isCall = mode === 'call';
const statusText = attached
? (isCall
? `Attached · call ${callState?.state || 'idle'}${callState?.turns ? ` · ${callState.turns} turns` : ''}`
: `Capturing · ${captureState?.turns || 0} utterance${captureState?.turns === 1 ? '' : 's'} transcribed`)
: (isCall ? 'Not attached — no call audio reaches PortOS.' : 'Not capturing — nothing is being transcribed.');
return (
<div className="mx-auto max-w-2xl space-y-4 p-4">
<header className="space-y-1">
<h1 className="flex items-center gap-2 text-xl font-semibold text-white"><PhoneCall size={20} />Call host</h1>
<p className="text-sm text-gray-400">
Keep this tab open on the Mac running PortOS. In <strong>Call</strong> mode it carries the
audio for a FaceTime Audio call: FaceTime plays into {DEFAULT_INPUT_LABEL}, and PortOS
answers through {DEFAULT_OUTPUT_LABEL}. In <strong>Capture system audio</strong> mode it only
listens to {DEFAULT_INPUT_LABEL} and transcribes — route any meeting app's output there via a
Multi-Output Device to capture it into the daily journal and the Brain inbox.
</p>
</header>
{/* The shared TabPills owns the roving tabindex + arrow-key contract;
`disabled` keeps a locked mode out of the arrow-key order too. */}
<TabPills
variant="pills"
tabs={[
{ id: 'call', label: 'Call', icon: PhoneCall, disabled: attached },
{ id: 'capture', label: 'Capture system audio', icon: Radio, disabled: attached },
]}
activeTab={mode}
onChange={setMode}
ariaLabel="Call host mode"
/>
{blocked && <p role="alert" className="rounded border border-port-error bg-port-error/10 p-3 text-sm text-port-error">{blocked}</p>}
{notice && <p role="status" className="rounded border border-port-warning bg-port-warning/10 p-3 text-sm text-port-warning">{notice}</p>}
<section className="space-y-3 rounded-lg border border-port-border bg-port-card p-4">
<div className="flex flex-wrap items-center gap-2">
{attached
? <button type="button" onClick={stop} className="flex min-h-[44px] items-center gap-2 rounded border border-port-border px-3 text-sm text-gray-300 hover:bg-port-border/50"><MicOff size={16} />{isCall ? 'Detach call host' : 'Stop capture'}</button>
: <button type="button" onClick={start} className="flex min-h-[44px] items-center gap-2 rounded bg-port-accent/20 px-3 text-sm text-port-accent hover:bg-port-accent/30"><Mic size={16} />{isCall ? 'Attach call host' : 'Start capture'}</button>}
{isCall && (
<button type="button" onClick={playTestTone} disabled={!attached} className="flex min-h-[44px] items-center gap-2 rounded border border-port-border px-3 text-sm text-gray-300 hover:bg-port-border/50 disabled:opacity-50"><Volume2 size={16} />Test tone</button>
)}
</div>
<div>
<p className="mb-1 text-xs text-gray-500" id="call-host-level-label">Input level</p>
<div aria-labelledby="call-host-level-label" role="meter" aria-valuenow={Math.round(level * 100)} aria-valuemin={0} aria-valuemax={100} className="h-2 w-full overflow-hidden rounded bg-port-bg">
<div className="h-full bg-port-accent transition-[width]" style={{ width: `${Math.min(100, Math.round(level * 140))}%` }} />
</div>
</div>
<p role="status" className="text-xs text-gray-400">{statusText}</p>
</section>
</div>
);
}