Skip to content

Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.

17.6 KB
Raw Blame
// Desktop press-to-talk: shortcut/Dock-mic press starts recording IMMEDIATELY into a compact
// bottom-center pill (stop icon + live equalizer + elapsed mm:ss); pressing again (or the pill's
// own stop icon) stops → transcribes → inserts the text directly into the target terminal
// (`pty.sendText(id, text, { enter: false })`, no Enter — nothing here ever auto-submits) and the
// pill closes.
//
// Two different ways to dismiss, not interchangeable:
//  - STOP (shortcut second-press / the pill's stop icon) — ends the recording and PROCESSES it
//    (transcribe → insert).
//  - CANCEL (Esc, or the ×/close affordances) — DISCARDS whatever was being recorded and closes
//    the whole overlay. Esc mid-recording is cancel, not stop, even though both are triggered by
//    a "press" — see handleClose vs the stopSignal branch below.
//
// No target selected at press time never records at all — see the warning-pill render branch.
import { useCallback, useEffect, useRef, useState } from 'react'
import { IconClose } from './icons'
import { createPortal } from 'react-dom'
import { equalizerBars } from '../lib/dictation-equalizer'
import { PcmCapture } from '../lib/pcm-capture'
import { micDeniedMessage, micPlatform } from '../lib/micPermission'
import { useSession } from '../session/session'
import { useSettings } from '../state/settings'
import { hasSpeechModel } from '@shared/speech'
import { deliverToComposer } from '../lib/chatComposerDictation'

/**
 * Where a take lands. `terminal` types it into the node's pane (`pty.sendText`, no Enter);
 * `chat-composer` hands it to ONE mounted ⌘M composer's textarea by its per-mount id
 * (lib/chatComposerDictation.ts) — a draft the user edits and sends, never typed into the pane.
 */
export type DictationTarget =
  | { kind: 'terminal'; nodeId: string; title: string }
  | { kind: 'chat-composer'; nodeId: string; composerId: string; title: string }

export interface DictationOverlayProps {
  target: DictationTarget | null
  /** Bumped by Canvas's toggleDictation on a second shortcut/Dock-mic press while the overlay is
   *  already open. A no-op prop change is the signal (any new value ≠ close-then-reopen — the
   *  overlay itself decides what a "press again" means from its own current phase: STOP while
   *  recording, CANCEL/close otherwise). */
  stopSignal: number
  onClose: () => void
  /** Routes to Settings → License (same section SpeechSection's own Pro upsells use) — invoked
   *  from the "See nodeterm Pro" action shown beside a Pro-model transcribe error. */
  onOpenLicense: () => void
}

/** A Pro-gated model was picked but the account isn't premium — see SpeechService.transcribeNow,
 *  which throws exactly this shape. Matched by substring since the model id is interpolated in. */
export function isProGateError(message: string): boolean {
  return message.includes('requires nodeterm Pro')
}

type Phase = 'idle' | 'recording' | 'transcribing'
/** 'no-model' = dictation is off (Whisper engine with the explicit None selection, issue #143) —
 *  never records, explains where to turn it on. 'warning' = no target at press time (never
 *  records). 'pill' = the compact recording/transcribing/error capsule — the whole surface for a
 *  terminal target, start to finish (the transcript is inserted straight into the terminal, so
 *  there is no editable card). */
export type DictationMode = 'no-model' | 'warning' | 'pill'

/** Pure — no side effects. Exported for its own unit coverage.
 *
 *  'no-model' wins over everything, target included: with a model missing the failure would
 *  otherwise surface only AFTER the user spoke a whole take (transcribe throws), which reads as
 *  a broken feature rather than an off one. */
export function dictationMode(target: DictationTarget | null, hasModel: boolean): DictationMode {
  if (!hasModel) return 'no-model'
  return target ? 'pill' : 'warning'
}

/** mm:ss, floored to whole seconds. Pure — no side effects. */
export function formatElapsed(ms: number): string {
  const total = Math.max(0, Math.floor(ms / 1000))
  const m = Math.floor(total / 60)
  const s = total % 60
  return `${m}:${s.toString().padStart(2, '0')}`
}

// Faster than the visual 12-bar window needs on its own — the point is to feed more DISTINCT
// samples into that scrolling window (see dictation-equalizer.ts) so the equalizer reads as live
// motion instead of a slow crawl. PcmCapture.level() (RMS of the last chunk) is cheap enough that
// polling twice as often here has no measurable cost.
const LEVEL_POLL_MS = 50 // ~20Hz

// Base64-encoded int16 PCM runs ~2.6 MB/min. The ws bridge caps a single frame at
// WS_MAX_PAYLOAD (8 MiB, see src/server/ws.ts) — a take left running past ~3 minutes would
// produce a transcribe payload big enough to blow that budget, and `ws` drops the WHOLE
// connection on an oversized frame, not just the one message. Auto-stop well under that line.
const MAX_RECORDING_MS = 150_000 // 2:30

/** Whether elapsed recording time has crossed the hard cap (see MAX_RECORDING_MS above). Pure. */
export function isAtRecordingCap(elapsedMs: number): boolean {
  return elapsedMs >= MAX_RECORDING_MS
}

/** How long the "no target selected" / "no model" warning pill stays up before it auto-dismisses. */
const NO_TARGET_DISMISS_MS = 2500

export function DictationOverlay({ target, stopSignal, onClose, onOpenLicense }: DictationOverlayProps) {
  const { api } = useSession()
  // The off state only exists for the local Whisper engine — cloud brings its own model. Read once
  // per mount (the overlay is a fresh mount per shortcut press, like `target`).
  const speech = useSettings((s) => s.settings.speech)
  const modelReady = speech.engine !== 'whisper' || hasSpeechModel(speech.model)

  const [phase, setPhase] = useState<Phase>('idle')
  const [error, setError] = useState<string | null>(null)
  const [elapsedMs, setElapsedMs] = useState(0)
  const [levelHistory, setLevelHistory] = useState<number[]>([])
  const [capped, setCapped] = useState(false)

  const captureRef = useRef<PcmCapture | null>(null)
  const consentAskedRef = useRef(false)
  const startedAtRef = useRef(0)
  const timerRef = useRef<ReturnType<typeof setInterval> | null>(null)
  const discardedRef = useRef(false)
  const onCloseRef = useRef(onClose)
  onCloseRef.current = onClose
  // Set false only in the unmount cleanup below. `dictationOpen`/`onClose` are shared with
  // Canvas across the whole overlay, not scoped to this mounted instance (only the React key/
  // nonce is) — so a stale instance (remounted-over by a fresh press while its own transcribe
  // was still in flight) must still deliver its transcript, but must NOT call onClose(): a
  // newer instance may be live and recording, and onClose() would unmount it out from under
  // itself. See the async-continuation sites in stopRecording below.
  const mountedRef = useRef(true)

  const clearTimer = useCallback(() => {
    if (timerRef.current) {
      clearInterval(timerRef.current)
      timerRef.current = null
    }
  }, [])

  // Belt-and-braces: if this component unmounts mid-recording (parent flips `open` without going
  // through handleClose — e.g. a project switch), the mic must not stay live.
  useEffect(() => {
    return () => {
      mountedRef.current = false
      clearTimer()
      captureRef.current?.cancel()
      captureRef.current = null
    }
  }, [clearTimer])

  const stopRecording = useCallback(async () => {
    clearTimer()
    const capture = captureRef.current
    captureRef.current = null
    if (!capture) return
    const pcm = capture.stop()
    setPhase('transcribing')
    try {
      const { text: transcribed } = await window.nodeTerminal.speech.transcribe(pcm)
      if (!target) {
        // Defensive only — recording never starts without a target (see the mount effect below).
        // Guarded like the terminal-insert path below: a superseded (remounted-over) instance
        // must not close an overlay a newer instance may now own.
        if (mountedRef.current) onClose()
        return
      }
      // An in-flight transcription can't be aborted — dropping the result honors the user's
      // dismissal (nothing may land after a cancel).
      if (discardedRef.current) return
      if (target.kind === 'chat-composer') {
        if (!deliverToComposer(target.composerId, transcribed)) {
          setError('Could not insert — the chat composer was closed.')
          setPhase('idle')
          return
        }
      } else {
        const ok = await api.pty.sendText(target.nodeId, transcribed, { enter: false })
        if (!ok) {
          setError('Could not insert — the terminal session is not available.')
          setPhase('idle')
          return
        }
      }
      // A stale (remounted-over) instance still delivers its own transcript into the
      // terminal — the target closure is per-instance and correct — but must not close the
      // parent overlay state: a newer instance (a fresh hold-to-talk press that arrived while
      // this one was still transcribing) may be live and recording in it right now.
      if (mountedRef.current) onClose()
    } catch (err) {
      setError(err instanceof Error ? err.message : 'Transcription failed.')
      setPhase('idle')
    }
  }, [clearTimer, target, api, onClose])

  const startRecording = useCallback(async () => {
    if (!target) return
    setError(null)
    setCapped(false)
    if (!consentAskedRef.current) {
      consentAskedRef.current = true
      try {
        const ok = await window.nodeTerminal.speech.micConsent()
        if (!ok) {
          setError(micDeniedMessage(micPlatform()))
          return
        }
      } catch (err) {
        setError(err instanceof Error ? err.message : 'Could not request microphone access.')
        return
      }
      // Closed during the consent round trip — never open the mic for an overlay that is gone.
      if (!mountedRef.current || discardedRef.current) return
    }

    // Owned BEFORE the mic finishes opening, so the unmount cleanup can cancel a start that is
    // still in flight (PcmCapture then releases the stream the moment it arrives). Opening takes
    // hundreds of ms (seconds with a Bluetooth headset), and hold-to-talk closes inside that
    // window on every quick tap, ⌘⌥<key> shortcut and window blur. Adopting the capture only
    // after `start()` resolved left exactly those takes with no owner: the mic recorded the room
    // until the 2:30 cap, and the transcript — whisper's "Thank you." over silence, or a
    // meeting's audio — was typed into the terminal.
    const capture = new PcmCapture()
    captureRef.current = capture
    let live: boolean
    try {
      live = await capture.start()
    } catch (err) {
      if (captureRef.current !== capture) return
      captureRef.current = null
      setError(err instanceof Error ? err.message : 'Could not start recording.')
      return
    }
    // Cancelled while opening (start() already released the stream), or superseded/unmounted.
    if (!live || captureRef.current !== capture || !mountedRef.current) {
      capture.cancel()
      return
    }

    startedAtRef.current = Date.now()
    setElapsedMs(0)
    setLevelHistory([])
    setPhase('recording')
    timerRef.current = setInterval(() => {
      const elapsed = Date.now() - startedAtRef.current
      setElapsedMs(elapsed)
      const level = captureRef.current?.level() ?? 0
      setLevelHistory((prev) => [...prev, level])
      if (isAtRecordingCap(elapsed)) {
        // clearTimer() inside stopRecording fires synchronously before any await, so this can't
        // re-enter on the next tick.
        setCapped(true)
        void stopRecording()
      }
    }, LEVEL_POLL_MS)
  }, [target, stopRecording])

  // Press-to-talk: a target means "start recording now" (this instance is a fresh mount each
  // time Canvas opens the overlay, and the target is frozen for its lifetime — see
  // DictationOverlayProps — so this is correctly mount-only). No target means the warning pill;
  // it never records, and auto-dismisses on its own.
  useEffect(() => {
    // Both warning shapes never record and auto-dismiss: no target, and dictation-off (no model —
    // recording a take that transcribe is guaranteed to refuse would waste the user's speech).
    if (!target || !modelReady) {
      const t = setTimeout(() => onCloseRef.current(), NO_TARGET_DISMISS_MS)
      return () => clearTimeout(t)
    }
    void startRecording()
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [])

  const handleClose = useCallback(() => {
    discardedRef.current = true
    if (phase === 'recording') {
      clearTimer()
      captureRef.current?.cancel()
      captureRef.current = null
    }
    onClose()
  }, [phase, clearTimer, onClose])

  // Esc closes (own listener, same pattern as ShortcutsPanel — only active while mounted, so it
  // never competes with Canvas's own Esc handling elsewhere). This is CANCEL, not stop — see the
  // file header.
  useEffect(() => {
    const onKey = (e: KeyboardEvent) => {
      if (e.key === 'Escape') {
        e.preventDefault()
        handleClose()
      }
    }
    window.addEventListener('keydown', onKey)
    return () => window.removeEventListener('keydown', onKey)
  }, [handleClose])

  // A second shortcut/Dock-mic press while this overlay is already open (Canvas bumps
  // `stopSignal` rather than unmounting us — see DictationOverlayProps). While actively
  // recording that means STOP (transcribe → insert); in every other phase (warning,
  // transcribing, an error pill) it means dismiss, same as Esc/×.
  const prevStopSignalRef = useRef(stopSignal)
  useEffect(() => {
    if (stopSignal === prevStopSignalRef.current) return
    prevStopSignalRef.current = stopSignal
    if (phase === 'recording') {
      void stopRecording()
    } else {
      handleClose()
    }
  }, [stopSignal, phase, stopRecording, handleClose])

  const mode = dictationMode(target, modelReady)

  // Desktop only: the browser bridge leaves `openMicSettings` absent (its permission is the
  // browser's own site setting), so the button simply does not render there.
  // Only macOS and Windows have a privacy page to open.
  const platform = micPlatform()
  const openMicSettings =
    platform === 'darwin' || platform === 'win32' ? window.nodeTerminal.speech?.openMicSettings : undefined
  const errorBlock = error && (
    <div className="dictation__error">
      <span>{error}</span>
      {openMicSettings && error === micDeniedMessage(platform) && (
        <button type="button" className="dictation__error-action" onClick={() => void openMicSettings().catch(() => undefined)}>
          Open microphone settings
        </button>
      )}
      {isProGateError(error) && (
        <button type="button" className="dictation__error-action" onClick={onOpenLicense}>
          See nodeterm Pro
        </button>
      )}
      <button type="button" className="dictation__close" title="Dismiss" aria-label="Dismiss" onClick={handleClose}>
        <IconClose />
      </button>
    </div>
  )

  if (mode === 'warning' || mode === 'no-model') {
    return createPortal(
      <div className="dictation dictation--warning nodrag nowheel" onMouseDown={(e) => e.stopPropagation()}>
        <span className="dictation__warning-text">
          {mode === 'no-model'
            ? 'Dictation is off — choose a Whisper model in Settings → Speech.'
            : 'Select a terminal node first.'}
        </span>
        <button type="button" className="dictation__close" title="Dismiss" aria-label="Dismiss" onClick={handleClose}>
          <IconClose />
        </button>
      </div>,
      document.body
    )
  }

  // mode === 'pill' — the whole surface for a terminal target (a transcribed take is inserted
  // straight into the terminal, so there is no editable card).
  return createPortal(
      <div className="dictation dictation--pill nodrag nowheel" onMouseDown={(e) => e.stopPropagation()}>
        {phase === 'recording' && (
          <div className="dictation__pill">
            {/* Live equalizer bars, centered so they grow up AND down from
                the middle — a scrolling window of real mic levels (newest at
                the right); silence keeps a visible baseline row. */}
            <div className="dictation__bars" aria-hidden="true">
              {equalizerBars(levelHistory, 7).map((h, i) => (
                <span key={i} className="dictation__bar" style={{ height: `${Math.round(h * 100)}%` }} />
              ))}
            </div>
            <span className="dictation__label">Dictating...</span>
            <span className="dictation__elapsed">{formatElapsed(elapsedMs)}</span>
            <span className="dictation__spacer" />
            <button
              type="button"
              className="dictation__pause"
              onClick={() => void stopRecording()}
              title="Stop recording — transcribes & inserts"
            >
              <PauseIcon />
            </button>
          </div>
        )}

        {phase === 'transcribing' && (
          <div className="dictation__transcribing">
            <span className="dictation__spinner" />
            <span>{capped ? 'Recording capped at 2:30 — transcribing…' : 'Transcribing…'}</span>
          </div>
        )}

        {phase === 'idle' && !error && (
          <div className="dictation__transcribing">
            <span className="dictation__spinner" />
            <span>Starting…</span>
          </div>
        )}

        {errorBlock}
      </div>,
      document.body
    )
}

function PauseIcon() {
  return (
    <svg width={16} height={16} viewBox="0 0 24 24" fill="currentColor">
      <rect x="6" y="4" width="4" height="16" rx="1.5" />
      <rect x="14" y="4" width="4" height="16" rx="1.5" />
    </svg>
  )
}