Enterprise agent platform for building, deploying, and operating AI teammates.
"use client"
import React, { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react"
import { Loader2, Mic, Square } from "lucide-react"
import { toast } from "@/components/ui/sonner"
import { useI18n } from "@/contexts/i18n-context"
import {
apiRequest,
getApiErrorMessage,
isJsonRecord,
parseApiResponse,
} from "@/lib/api-wrapper"
import { cn, getApiUrl, getUploadApiUrl } from "@/lib/utils"
import { supportsBatchAsr } from "@/lib/models"
export type VoiceTarget = HTMLInputElement | HTMLTextAreaElement | HTMLElement
export type VoiceStatus = "idle" | "recording" | "transcribing"
interface UseVoiceInputControlsOptions {
autoRefresh?: boolean
enabled?: boolean
}
const TEXT_INPUT_TYPES = new Set(["", "email", "search", "tel", "text", "url"])
const useIsomorphicLayoutEffect =
typeof window === "undefined" ? useEffect : useLayoutEffect
const SENSITIVE_FIELD_PATTERN =
/(api[_-]?key|authorization|bearer|card[_-]?number|cardnumber|client[_-]?id|client[_-]?secret|credit[_-]?card|csc|cvc|cvv|one[_-]?time|otp|passcode|password|pin|security[_-]?code|securitycode|secret|social[_-]?security|ssn|token)/i
const SENSITIVE_AUTOCOMPLETE_TOKENS = new Set([
"cc-additional-name",
"cc-csc",
"cc-exp",
"cc-exp-month",
"cc-exp-year",
"cc-family-name",
"cc-given-name",
"cc-name",
"cc-number",
"cc-type",
"current-password",
"new-password",
"one-time-code",
])
function clamp(value: number, min: number, max: number): number {
return Math.min(Math.max(value, min), max)
}
function isElementHidden(element: HTMLElement): boolean {
const rect = element.getBoundingClientRect()
return rect.width <= 0 || rect.height <= 0
}
function getPositionAnchor(target: VoiceTarget): HTMLElement {
const explicitAnchor = target.closest<HTMLElement>("[data-voice-input-anchor]")
if (explicitAnchor && !isElementHidden(explicitAnchor)) {
return explicitAnchor
}
const canUseEnclosingAnchor =
target.isContentEditable || target instanceof HTMLTextAreaElement
if (!canUseEnclosingAnchor) {
return target
}
const targetRect = target.getBoundingClientRect()
let current = target.parentElement
while (current && current !== document.body) {
if (!isElementHidden(current)) {
const rect = current.getBoundingClientRect()
const containsTarget =
rect.left <= targetRect.left &&
rect.right >= targetRect.right &&
rect.top <= targetRect.top &&
rect.bottom >= targetRect.bottom
const expandsTarget =
rect.width >= targetRect.width + 48 ||
rect.height >= targetRect.height + 24
if (containsTarget && expandsTarget) {
return current
}
}
current = current.parentElement
}
return target
}
function fieldFingerprint(element: HTMLElement): string {
return [
element.getAttribute("aria-label"),
element.getAttribute("id"),
element.getAttribute("name"),
element.getAttribute("placeholder"),
]
.filter(Boolean)
.join(" ")
}
function hasSensitiveAutocomplete(element: HTMLElement): boolean {
const autocomplete = element.getAttribute("autocomplete")
if (!autocomplete) return false
return autocomplete
.toLowerCase()
.split(/\s+/)
.some((token) => SENSITIVE_AUTOCOMPLETE_TOKENS.has(token))
}
function isVoiceEligibleTarget(target: EventTarget | null): target is VoiceTarget {
if (!(target instanceof HTMLElement)) return false
if (target.closest("[data-voice-input-root]")) return false
if (target.closest("[data-voice-input='false']")) return false
if (target.getAttribute("data-voice-input") !== "true") return false
if (target.getAttribute("aria-readonly") === "true") return false
if (SENSITIVE_FIELD_PATTERN.test(fieldFingerprint(target))) return false
if (hasSensitiveAutocomplete(target)) return false
if (target instanceof HTMLInputElement) {
return (
TEXT_INPUT_TYPES.has(target.type) &&
!target.disabled &&
!target.readOnly &&
!isElementHidden(target)
)
}
if (target instanceof HTMLTextAreaElement) {
return !target.disabled && !target.readOnly && !isElementHidden(target)
}
return (
target.isContentEditable &&
!isElementHidden(target)
)
}
function isInsideActiveVoiceArea(
eventTarget: EventTarget | null,
currentTarget: VoiceTarget | null
): boolean {
if (!(eventTarget instanceof Element) || !currentTarget) return false
if (eventTarget.closest("[data-voice-input-root]")) return true
const anchor = getPositionAnchor(currentTarget)
return currentTarget.contains(eventTarget) || anchor.contains(eventTarget)
}
function dispatchInputEvents(target: HTMLElement, data: string): void {
try {
target.dispatchEvent(
new InputEvent("input", {
bubbles: true,
data,
inputType: "insertText",
})
)
} catch {
target.dispatchEvent(new Event("input", { bubbles: true }))
}
target.dispatchEvent(new Event("change", { bubbles: true }))
}
function setNativeValue(
target: HTMLInputElement | HTMLTextAreaElement,
value: string
): void {
const prototype =
target instanceof HTMLTextAreaElement
? HTMLTextAreaElement.prototype
: HTMLInputElement.prototype
const descriptor = Object.getOwnPropertyDescriptor(prototype, "value")
descriptor?.set?.call(target, value)
}
function withWordBoundary(
existing: string,
start: number,
end: number,
rawText: string
): string {
const text = rawText.trim()
if (!text) return ""
const before = existing.slice(0, start)
const after = existing.slice(end)
const needsLeadingSpace =
before.length > 0 && !/\s$/.test(before) && !/^[,.;:!?,。!?、]/.test(text)
const needsTrailingSpace =
after.length > 0 && !/^\s/.test(after) && !/[([{(【「『]$/.test(text)
return `${needsLeadingSpace ? " " : ""}${text}${needsTrailingSpace ? " " : ""}`
}
function insertTranscribedText(target: VoiceTarget, rawText: string): void {
const text = rawText.trim()
if (!text) return
target.focus()
if (target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement) {
const value = target.value
const start = target.selectionStart ?? value.length
const end = target.selectionEnd ?? start
const insertion = withWordBoundary(value, start, end, text)
const nextValue = `${value.slice(0, start)}${insertion}${value.slice(end)}`
setNativeValue(target, nextValue)
const cursor = start + insertion.length
target.setSelectionRange(cursor, cursor)
dispatchInputEvents(target, insertion)
return
}
const selection = window.getSelection()
const currentText = target.textContent || ""
const insertion =
currentText.length > 0 && !/\s$/.test(currentText) ? ` ${text}` : text
if (
selection &&
selection.rangeCount > 0 &&
selection.anchorNode &&
target.contains(selection.anchorNode)
) {
document.execCommand("insertText", false, insertion)
} else {
target.appendChild(document.createTextNode(insertion))
}
dispatchInputEvents(target, insertion)
}
function chooseMimeType(): string | undefined {
if (typeof MediaRecorder === "undefined") return undefined
const candidates = [
"audio/webm;codecs=opus",
"audio/webm",
"audio/mp4",
"audio/ogg;codecs=opus",
]
return candidates.find((type) => MediaRecorder.isTypeSupported(type))
}
function extensionForMimeType(mimeType: string): string {
const normalized = mimeType.toLowerCase()
if (normalized.includes("mp4")) return "m4a"
if (normalized.includes("ogg")) return "ogg"
if (normalized.includes("wav")) return "wav"
return "webm"
}
export function useVoiceInputControls(
{ autoRefresh = true, enabled = true }: UseVoiceInputControlsOptions = {}
) {
const { t } = useI18n()
const [hasAsrModel, setHasAsrModel] = useState(false)
const [status, setStatus] = useState<VoiceStatus>("idle")
const targetRef = useRef<VoiceTarget | null>(null)
const recorderRef = useRef<MediaRecorder | null>(null)
const recordingTargetRef = useRef<VoiceTarget | null>(null)
const streamRef = useRef<MediaStream | null>(null)
const chunksRef = useRef<BlobPart[]>([])
const lastAvailabilityFetchRef = useRef(0)
const availabilityRequestRef = useRef(0)
const enabledRef = useRef(enabled)
const mountedRef = useRef(false)
const capabilityEpochRef = useRef(0)
const isCurrentCapability = useCallback(
(capabilityEpoch: number) =>
mountedRef.current &&
enabledRef.current &&
capabilityEpoch === capabilityEpochRef.current,
[]
)
const stopStream = useCallback(() => {
streamRef.current?.getTracks().forEach((track) => track.stop())
streamRef.current = null
}, [])
const retireCapability = useCallback(
({ clearAvailability }: { clearAvailability: boolean }) => {
capabilityEpochRef.current += 1
availabilityRequestRef.current += 1
targetRef.current = null
recordingTargetRef.current = null
chunksRef.current = []
const recorder = recorderRef.current
recorderRef.current = null
if (recorder && recorder.state !== "inactive") {
recorder.stop()
}
stopStream()
if (mountedRef.current) {
if (clearAvailability) {
setHasAsrModel(false)
}
setStatus("idle")
}
},
[stopStream]
)
useIsomorphicLayoutEffect(() => {
mountedRef.current = true
return () => {
mountedRef.current = false
enabledRef.current = false
retireCapability({ clearAvailability: false })
}
}, [retireCapability])
useIsomorphicLayoutEffect(() => {
if (enabledRef.current === enabled) return
enabledRef.current = enabled
retireCapability({ clearAvailability: !enabled })
}, [enabled, retireCapability])
const refreshAvailability = useCallback(async () => {
const capabilityEpoch = capabilityEpochRef.current
const requestId = ++availabilityRequestRef.current
if (!isCurrentCapability(capabilityEpoch)) return
lastAvailabilityFetchRef.current = Date.now()
try {
const response = await apiRequest(
`${getApiUrl()}/api/models/?category=speech&limit=1000`
)
if (
!isCurrentCapability(capabilityEpoch)
|| requestId !== availabilityRequestRef.current
) {
return
}
if (!response.ok) {
setHasAsrModel(false)
return
}
const models = await response.json()
if (
!isCurrentCapability(capabilityEpoch)
|| requestId !== availabilityRequestRef.current
) {
return
}
const available =
Array.isArray(models) &&
models.some(supportsBatchAsr)
setHasAsrModel(available)
} catch {
if (
isCurrentCapability(capabilityEpoch)
&& requestId === availabilityRequestRef.current
) {
setHasAsrModel(false)
}
}
}, [isCurrentCapability])
useEffect(() => {
if (!enabled) {
return
}
if (autoRefresh) {
refreshAvailability()
}
}, [autoRefresh, enabled, refreshAvailability])
const refreshAvailabilityIfStale = useCallback(() => {
if (!enabledRef.current) return
if (Date.now() - lastAvailabilityFetchRef.current > 30_000) {
refreshAvailability()
}
}, [refreshAvailability])
const transcribeBlob = useCallback(
async (
blob: Blob,
target: VoiceTarget | null,
capabilityEpoch: number
) => {
if (!isCurrentCapability(capabilityEpoch) || !target) return
if (blob.size === 0) {
toast.error(t("voiceInput.errors.emptyAudio"))
return
}
setStatus("transcribing")
try {
const mimeType = blob.type || "audio/webm"
const extension = extensionForMimeType(mimeType)
const file = new File([blob], `voice-input.${extension}`, {
type: mimeType,
})
const formData = new FormData()
formData.append("file", file)
if (!isCurrentCapability(capabilityEpoch)) return
const response = await apiRequest(
`${getUploadApiUrl()}/api/models/speech/transcribe`,
{
method: "POST",
body: formData,
}
)
if (!isCurrentCapability(capabilityEpoch)) return
const parsed = await parseApiResponse(response)
if (!isCurrentCapability(capabilityEpoch)) return
if (!response.ok) {
throw new Error(
getApiErrorMessage(response, parsed, t("voiceInput.errors.failed"))
)
}
const text =
isJsonRecord(parsed.data) && typeof parsed.data.text === "string"
? parsed.data.text
: ""
if (!text.trim()) {
toast.error(t("voiceInput.errors.noText"))
return
}
insertTranscribedText(target, text)
} catch (error) {
if (isCurrentCapability(capabilityEpoch)) {
toast.error(
error instanceof Error ? error.message : t("voiceInput.errors.failed")
)
}
} finally {
if (isCurrentCapability(capabilityEpoch)) {
setStatus("idle")
}
}
},
[isCurrentCapability, t]
)
const startRecording = useCallback(
async (target?: VoiceTarget | null) => {
const capabilityEpoch = capabilityEpochRef.current
const recordingTarget = target ?? targetRef.current
if (
!isCurrentCapability(capabilityEpoch)
|| !recordingTarget
|| status !== "idle"
) return
if (
!navigator.mediaDevices?.getUserMedia ||
typeof MediaRecorder === "undefined"
) {
toast.error(t("voiceInput.errors.unsupported"))
return
}
targetRef.current = recordingTarget
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
if (!isCurrentCapability(capabilityEpoch)) {
stream.getTracks().forEach((track) => track.stop())
return
}
streamRef.current = stream
chunksRef.current = []
recordingTargetRef.current = recordingTarget
const mimeType = chooseMimeType()
const recorder = new MediaRecorder(
stream,
mimeType ? { mimeType } : undefined
)
recorderRef.current = recorder
recorder.ondataavailable = (event) => {
if (
!isCurrentCapability(capabilityEpoch)
|| recorderRef.current !== recorder
) return
if (event.data.size > 0) {
chunksRef.current.push(event.data)
}
}
recorder.onstop = () => {
if (
!isCurrentCapability(capabilityEpoch)
|| recorderRef.current !== recorder
) return
const blob = new Blob(chunksRef.current, {
type: recorder.mimeType || mimeType || "audio/webm",
})
chunksRef.current = []
recorderRef.current = null
const target = recordingTargetRef.current
recordingTargetRef.current = null
stopStream()
transcribeBlob(blob, target, capabilityEpoch)
}
recorder.onerror = () => {
if (
!isCurrentCapability(capabilityEpoch)
|| recorderRef.current !== recorder
) return
stopStream()
recorderRef.current = null
recordingTargetRef.current = null
chunksRef.current = []
setStatus("idle")
toast.error(t("voiceInput.errors.failed"))
}
recorder.start()
setStatus("recording")
} catch (error) {
if (!isCurrentCapability(capabilityEpoch)) return
stopStream()
recordingTargetRef.current = null
setStatus("idle")
toast.error(
error instanceof DOMException && error.name === "NotAllowedError"
? t("voiceInput.errors.permissionDenied")
: t("voiceInput.errors.failed")
)
}
},
[isCurrentCapability, status, stopStream, t, transcribeBlob]
)
const stopRecording = useCallback(() => {
const recorder = recorderRef.current
if (recorder && recorder.state !== "inactive") {
recorder.stop()
}
}, [])
return {
hasAsrModel,
refreshAvailability,
refreshAvailabilityIfStale,
startRecording,
status,
stopRecording,
}
}
export function VoiceInputController() {
const { t } = useI18n()
const {
hasAsrModel,
refreshAvailabilityIfStale,
startRecording,
status,
stopRecording,
} = useVoiceInputControls({ autoRefresh: false })
const [activeTarget, setActiveTarget] = useState<VoiceTarget | null>(null)
const [position, setPosition] = useState<{ top: number; left: number } | null>(
null
)
const targetRef = useRef<VoiceTarget | null>(null)
const updatePosition = useCallback((target = targetRef.current) => {
if (!target || isElementHidden(target)) {
setPosition(null)
return
}
const anchor = getPositionAnchor(target)
const rect = anchor.getBoundingClientRect()
if (
rect.bottom < 0 ||
rect.top > window.innerHeight ||
rect.right < 0 ||
rect.left > window.innerWidth
) {
setPosition(null)
return
}
const buttonSize = 32
const viewportPadding = 8
const anchorInset = 8
const actionRowRightInset = 72
const actionRowBottomInset = 24
const useActionRowPosition = anchor !== target && rect.height >= 88
const regularTop =
rect.height > buttonSize + anchorInset * 2
? rect.top + anchorInset
: rect.top + (rect.height - buttonSize) / 2
const anchorTop = useActionRowPosition
? rect.bottom - buttonSize - actionRowBottomInset
: regularTop
const outsideLeft = rect.right + 6
const insideLeft =
rect.right -
buttonSize -
(useActionRowPosition ? actionRowRightInset : anchorInset)
const hasOutsideSpace =
anchor === target &&
outsideLeft + buttonSize <= window.innerWidth - viewportPadding
const left = clamp(
hasOutsideSpace ? outsideLeft : insideLeft,
viewportPadding,
window.innerWidth - buttonSize - viewportPadding
)
const top = clamp(
anchorTop,
viewportPadding,
window.innerHeight - buttonSize - viewportPadding
)
setPosition({ top, left })
}, [])
const activateTarget = useCallback(
(target: VoiceTarget) => {
if (status !== "idle") return
refreshAvailabilityIfStale()
targetRef.current = target
setActiveTarget(target)
updatePosition(target)
},
[refreshAvailabilityIfStale, status, updatePosition]
)
const clearTarget = useCallback(() => {
if (status !== "idle") return
setActiveTarget(null)
targetRef.current = null
setPosition(null)
}, [status])
useEffect(() => {
const focusedTarget = document.activeElement
if (isVoiceEligibleTarget(focusedTarget)) {
activateTarget(focusedTarget)
}
const handleFocusIn = (event: FocusEvent) => {
const target = event.target
if (!isVoiceEligibleTarget(target)) {
if (!isInsideActiveVoiceArea(target, targetRef.current)) clearTarget()
return
}
activateTarget(target)
}
const handlePointerOver = (event: PointerEvent) => {
const target = event.target
if (isVoiceEligibleTarget(target)) {
activateTarget(target)
return
}
if (isInsideActiveVoiceArea(target, targetRef.current)) return
if (document.activeElement === targetRef.current) return
clearTarget()
}
document.addEventListener("focusin", handleFocusIn)
document.addEventListener("pointerover", handlePointerOver)
return () => {
document.removeEventListener("focusin", handleFocusIn)
document.removeEventListener("pointerover", handlePointerOver)
}
}, [activateTarget, clearTarget])
useEffect(() => {
let frameId: number | null = null
const reposition = () => {
if (frameId !== null) return
frameId = window.requestAnimationFrame(() => {
frameId = null
updatePosition()
})
}
window.addEventListener("resize", reposition)
document.addEventListener("scroll", reposition, true)
return () => {
window.removeEventListener("resize", reposition)
document.removeEventListener("scroll", reposition, true)
if (frameId !== null) {
window.cancelAnimationFrame(frameId)
}
}
}, [updatePosition])
const visible = hasAsrModel && !!activeTarget && !!position
if (!visible || !position) {
return null
}
const label =
status === "recording"
? t("voiceInput.stop")
: status === "transcribing"
? t("voiceInput.transcribing")
: t("voiceInput.start")
return (
<button
type="button"
data-voice-input-root
aria-label={label}
title={label}
className={cn(
"fixed z-[70] inline-flex h-8 w-8 items-center justify-center rounded-full border shadow-md transition-all",
"focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2",
status === "recording"
? "border-red-400 bg-red-500 text-white hover:bg-red-600"
: "border-border bg-background text-muted-foreground hover:bg-accent hover:text-foreground",
status === "transcribing" && "cursor-wait opacity-80"
)}
style={{ top: position.top, left: position.left }}
disabled={status === "transcribing"}
onMouseDown={(event) => event.preventDefault()}
onClick={() => {
if (status === "recording") {
stopRecording()
} else if (status === "idle") {
startRecording(activeTarget)
}
}}
>
{status === "recording" ? (
<Square className="h-3.5 w-3.5 fill-current" />
) : status === "transcribing" ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : (
<Mic className="h-4 w-4" />
)}
</button>
)
}