Skip to content
TypeScript

feat(web): opt into environment dictation on this device (#30623)

Julius Marmingea9a1b8c4be75
diff --git a/apps/server/src/speech/SpeechModelStore.test.ts b/apps/server/src/speech/SpeechModelStore.test.ts
index dda2f3cad4e..e08a4cf022f 100644
--- a/apps/server/src/speech/SpeechModelStore.test.ts
+++ b/apps/server/src/speech/SpeechModelStore.test.ts
@@ -13,7 +13,6 @@ import * as Stream from "effect/Stream";
import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http";
import * as ServerConfig from "../config.ts";
-import { getSpeechModel, type SpeechModel } from "./model.ts";
import * as SpeechModelStore from "./SpeechModelStore.ts";
const bytes = Uint8Array.from({ length: 64 }, (_, index) => (index * 7) % 256);
@@ -21,8 +20,9 @@ const SPLIT = 20;
const harness = Effect.gen(function* () {
const digest = yield* Digest.Digest;
- const model: SpeechModel = {
- ...getSpeechModel("handy-computer/whisper-tiny-gguf")!,
+ const model = {
+ id: "fixture/speech-model",
+ revision: "a".repeat(40),
filename: "model.gguf",
size: bytes.length,
sha256: Hex.encode(yield* digest.digest(Digest.sha256, bytes).pipe(Effect.orDie)),
@@ -121,10 +121,8 @@ it.effect("resumes an interrupted transfer, replaces a corrupt same-size file, a
undefined,
`bytes=${SPLIT}-`,
]);
- expect(h.requests.map((request) => request.url)).toEqual([
- "https://huggingface.co/handy-computer/whisper-tiny-gguf/resolve/6687f30c99641ee265df421e582354adbc8848fc/model.gguf",
- "https://huggingface.co/handy-computer/whisper-tiny-gguf/resolve/6687f30c99641ee265df421e582354adbc8848fc/model.gguf",
- ]);
+ const pinnedUrl = `https://huggingface.co/${h.model.id}/resolve/${h.model.revision}/${h.model.filename}`;
+ expect(h.requests.map((request) => request.url)).toEqual([pinnedUrl, pinnedUrl]);
expect(progress.at(0)).toBe(0);
expect(progress.at(-1)).toBe(bytes.length);
expect([...(yield* h.fs.readFile(installed))]).toEqual([...bytes]);
@@ -134,6 +132,9 @@ it.effect("resumes an interrupted transfer, replaces a corrupt same-size file, a
]);
expect(yield* h.store.isReady(h.model)).toBe(true);
expect(yield* h.store.installedPath(h.model)).toBe(installed);
+ const requestsBeforeReuse = h.requests.length;
+ expect(yield* h.store.download(h.model)).toBe(installed);
+ expect(h.requests).toHaveLength(requestsBeforeReuse);
}).pipe(Effect.provide(dependencies)),
);
diff --git a/apps/server/src/speech/SpeechModelStore.ts b/apps/server/src/speech/SpeechModelStore.ts
index b85d5b47f1f..99f3fe6ca5b 100644
--- a/apps/server/src/speech/SpeechModelStore.ts
+++ b/apps/server/src/speech/SpeechModelStore.ts
@@ -17,6 +17,8 @@ import * as Stream from "effect/Stream";
import * as ServerConfig from "../config.ts";
import type { SpeechModel } from "./model.ts";
+type ModelFile = Pick<SpeechModel, "id" | "filename" | "revision" | "size" | "sha256">;
+
class SpeechModelDownloadError extends Schema.TaggedError<SpeechModelDownloadError>()(
"SpeechModelDownloadError",
{
@@ -39,17 +41,17 @@ export class SpeechModelStore extends Context.Service<
SpeechModelStore,
{
/** Cheap size check for status polls; never hashes the file. */
- readonly isReady: (model: SpeechModel) => Effect.Effect<boolean>;
+ readonly isReady: (model: ModelFile) => Effect.Effect<boolean>;
/** The installed file once its SHA-256 has been verified, hashing it on first use. */
- readonly installedPath: (model: SpeechModel) => Effect.Effect<string | undefined>;
+ readonly installedPath: (model: ModelFile) => Effect.Effect<string | undefined>;
readonly download: (
- model: SpeechModel,
+ model: ModelFile,
progress?: SpeechModelDownloadProgress,
) => Effect.Effect<
string,
SpeechModelDownloadError | PlatformError | HttpClientError.HttpClientError
>;
- readonly remove: (model: SpeechModel) => Effect.Effect<void, PlatformError>;
+ readonly remove: (model: ModelFile) => Effect.Effect<void, PlatformError>;
}
>()("t3/speech/SpeechModelStore") {}
@@ -70,11 +72,11 @@ const make = Effect.gen(function* () {
yield* fs.remove(path.join(directory, name), { force: true }).pipe(Effect.ignore);
}
}
- const modelPath = (model: SpeechModel) => path.join(directory, model.filename);
+ const modelPath = (model: ModelFile) => path.join(directory, model.filename);
// Files whose digest failed, by identity, so status polls stop calling them ready.
const corrupt = new Map<string, string>();
- const identity = (model: SpeechModel, info: FileSystem.File.Info) =>
+ const identity = (model: ModelFile, info: FileSystem.File.Info) =>
[
model.sha256,
info.size,
@@ -85,7 +87,7 @@ const make = Effect.gen(function* () {
const statFile = (file: string) => fs.stat(file).pipe(Effect.orElseSucceed(() => undefined));
- const expectedFile = (model: SpeechModel, file: string) =>
+ const expectedFile = (model: ModelFile, file: string) =>
statFile(file).pipe(
Effect.map((info) =>
info?.type === "File" && Number(info.size) === model.size ? info : undefined,
@@ -93,7 +95,7 @@ const make = Effect.gen(function* () {
);
const digestMatches = Effect.fn("SpeechModelStore.digest")(
- function* (model: SpeechModel, file: string) {
+ function* (model: ModelFile, file: string) {
const hash = NodeCrypto.createHash("sha256");
yield* fs
.stream(file)
@@ -105,7 +107,7 @@ const make = Effect.gen(function* () {
// A verified file is trusted while its size, mtime, device and inode are unchanged.
const verifiedPath = Effect.fn("SpeechModelStore.verifiedPath")(function* (
- model: SpeechModel,
+ model: ModelFile,
file: string,
onVerificationStart?: () => void,
) {
@@ -127,7 +129,7 @@ const make = Effect.gen(function* () {
});
const fetchInto = Effect.fn("SpeechModelStore.fetch")(function* (
- model: SpeechModel,
+ model: ModelFile,
url: string,
partialPath: string,
offset: number,
diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts
index 3c93ba62f31..972c401ce30 100644
--- a/apps/server/src/speech/SpeechService.test.ts
+++ b/apps/server/src/speech/SpeechService.test.ts
@@ -10,19 +10,22 @@ import * as Result from "effect/Result";
import * as Scheduler from "effect/Scheduler";
import * as Scope from "effect/Scope";
import * as Exit from "effect/Exit";
+import * as PlatformError from "effect/PlatformError";
import * as HostProcess from "@t3tools/shared/HostProcess";
import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS } from "@t3tools/contracts";
import * as ServerSettings from "../serverSettings.ts";
import { listNativeSpeechGpuDevices } from "./native.ts";
import * as SpeechModelStore from "./SpeechModelStore.ts";
import * as SpeechService from "./SpeechService.ts";
+import { SILERO_VAD } from "./silero.ts";
const native = vi.hoisted(() => ({
backend: "Vulkan0",
supportsStreaming: true,
+ supportsLongForm: false,
supportsTranslation: true,
supportsVocabulary: false,
- begin: vi.fn(async (_language?: string, _vocabulary?: readonly string[]) => {}),
+ begin: vi.fn(async (_options?: unknown, _filterSilence?: boolean, _vadPath?: string) => {}),
feed: vi.fn(async (_pcm: Float32Array) => ({
revision: 1,
text: { committed: "", tentative: "hello" },
@@ -66,13 +69,13 @@ const modelDefinitions = vi.hoisted(() => [
name: "fallback",
description: "fallback",
size: 8,
- languages: ["en"],
+ languages: ["en", "fr"],
accuracy: 2,
speed: 2,
recommended: false,
supportsStreaming: false,
supportsLanguageDetection: false,
- supportsTranslation: false,
+ supportsTranslation: true,
},
]);
vi.mock("./native.ts", () => ({
@@ -112,8 +115,10 @@ const pcm = () => new Uint8Array(new Float32Array([0.25]).buffer);
beforeEach(() => {
vi.clearAllMocks();
loadNative.mockReset();
+ native.finish.mockReset();
native.backend = "Vulkan0";
native.supportsVocabulary = false;
+ native.supportsStreaming = true;
readyModels.clear();
readyModels.add("test-model");
readyModels.add("fallback-model");
@@ -146,6 +151,21 @@ it.effect("prepares a batch model before audio arrives and reuses it for transcr
yield* speech.prepareModel;
expect(loadNative).toHaveBeenCalledTimes(1);
expect(native.transcribe).not.toHaveBeenCalled();
+ native.supportsStreaming = false;
+ yield* Effect.gen(function* () {
+ const stream = yield* speech.startStream({
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ speechLanguage: "fr",
+ speechTranslateToEnglish: true,
+ speechSilenceFiltering: false,
+ });
+ expect(native.begin).toHaveBeenLastCalledWith(
+ { timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
+ false,
+ "test.gguf",
+ );
+ expect(yield* stream.finish).toBe("hello");
+ }).pipe(Effect.scoped);
expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
expect(loadNative).toHaveBeenCalledTimes(1);
}).pipe(Effect.provide(layer)),
@@ -223,28 +243,112 @@ it.effect("keeps a detached download exclusive until the transfer finishes", ()
const speech = yield* SpeechService.SpeechService;
const started = yield* Deferred.make<void>();
const finish = yield* Deferred.make<void>();
- download.mockImplementationOnce(() =>
- Deferred.succeed(started, undefined).pipe(
- Effect.andThen(Deferred.await(finish)),
- Effect.as("test.gguf"),
- ),
- );
+ const helperStarted = yield* Deferred.make<void>();
+ let helperStartedDuringInstall = false;
+ let helperInterrupted = false;
+ download
+ .mockImplementationOnce(() =>
+ Deferred.succeed(started, undefined).pipe(
+ Effect.andThen(Deferred.await(finish)),
+ Effect.as("test.gguf"),
+ ),
+ )
+ .mockImplementationOnce(() =>
+ Deferred.isDone(finish).pipe(
+ Effect.tap((done) =>
+ Effect.sync(() => {
+ helperStartedDuringInstall = !done;
+ }),
+ ),
+ Effect.andThen(Deferred.succeed(helperStarted, undefined)),
+ Effect.andThen(Effect.never),
+ Effect.onInterrupt(() =>
+ Effect.sync(() => {
+ helperInterrupted = true;
+ }),
+ ),
+ ),
+ );
const request = yield* speech.downloadModel("test-model").pipe(Effect.forkChild);
yield* Deferred.await(started);
+ yield* Deferred.await(helperStarted);
yield* Fiber.interrupt(request);
+ yield* Deferred.succeed(finish, undefined);
+ yield* TestClock.adjust("16 seconds");
+ expect(helperInterrupted).toBe(false);
const removed = yield* Effect.result(speech.removeModel("test-model"));
expect(Result.isFailure(removed) && removed.failure).toMatchObject({ _tag: "SpeechBusyError" });
const join = yield* speech
.downloadModel("test-model")
.pipe(Effect.forkChild({ startImmediately: true }));
- yield* Deferred.succeed(finish, undefined);
+ yield* TestClock.adjust("2 minutes");
+ expect(helperInterrupted).toBe(true);
yield* Fiber.join(join);
- expect(download).toHaveBeenCalledTimes(1);
+ expect(download.mock.calls.map(([model]) => model.id)).toEqual(["test-model", SILERO_VAD.id]);
+ expect(helperStartedDuringInstall).toBe(true);
yield* speech.removeModel("test-model");
expect(readyModels.has("test-model")).toBe(false);
}).pipe(Effect.provide(layer)),
);
+it.effect("bounds missing Silero recovery during recording and continues with raw audio", () =>
+ Effect.gen(function* () {
+ const helperStarted = yield* Deferred.make<void>();
+ const helperInterrupted = yield* Deferred.make<void>();
+ download.mockImplementationOnce(() =>
+ Deferred.succeed(helperStarted, undefined).pipe(
+ Effect.andThen(Effect.never),
+ Effect.onInterrupt(() => Deferred.succeed(helperInterrupted, undefined)),
+ ),
+ );
+ const speech = yield* SpeechService.SpeechService;
+ const recording = yield* speech
+ .startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
+ .pipe(Effect.forkChild);
+ yield* Deferred.await(helperStarted);
+ yield* TestClock.adjust("14999 millis");
+ expect(native.begin).not.toHaveBeenCalled();
+ yield* TestClock.adjust("1 millis");
+ yield* Deferred.await(helperInterrupted);
+ const stream = yield* Fiber.join(recording);
+ expect(native.begin).toHaveBeenLastCalledWith(expect.anything(), true, undefined);
+ yield* stream.feed(pcm());
+ expect(yield* stream.finish).toBe("hello");
+ }).pipe(Effect.scoped, Effect.provide(layer)),
+);
+
+it.effect("keeps the installed speech model usable when optional Silero acquisition fails", () =>
+ Effect.gen(function* () {
+ download
+ .mockImplementationOnce(() => Effect.succeed("test.gguf"))
+ .mockImplementationOnce(() =>
+ Effect.fail(
+ new PlatformError.PlatformError(
+ new PlatformError.SystemError({
+ _tag: "NotFound",
+ module: "FileSystem",
+ method: "open",
+ }),
+ ),
+ ),
+ );
+ const speech = yield* SpeechService.SpeechService;
+ const installed = yield* speech.downloadModel("test-model");
+ expect(installed.models.find((model) => model.id === "test-model")?.state).toBe("installed");
+ expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
+ yield* Effect.scoped(
+ Effect.gen(function* () {
+ const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
+ expect(yield* stream.finish).toBe("hello");
+ }),
+ );
+ expect(download).toHaveBeenCalledTimes(2);
+ yield* TestClock.adjust("10 minutes");
+ expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
+ expect(download).toHaveBeenCalledTimes(3);
+ }).pipe(Effect.provide(layer)),
+);
+
it.effect(
"allows another model operation after an already-installed download completes immediately",
() =>
@@ -333,7 +437,11 @@ it.effect("uses vocabulary only for supported transcription and always applies a
text: { committed: vocabulary ? "git hub" : "GitHub", tentative: "Bob" },
});
expect(yield* stream.finish).toBe(vocabulary ? "git hub and Bob" : "GitHub and Bob");
- expect(native.begin.mock.lastCall).toEqual(["fr", vocabulary]);
+ expect(native.begin.mock.lastCall).toEqual([
+ { timestamps: "none", language: "fr", ...(vocabulary ? { vocabulary } : {}) },
+ true,
+ "test.gguf",
+ ]);
}).pipe(Effect.scoped, Effect.provide(layer));
}
}
@@ -354,6 +462,58 @@ it.effect("unloads after the chosen idle period and reloads on the next recordin
}).pipe(Effect.provide(layer)),
);
+it.effect.each([
+ { cue: "um", remove: true, text: "Make it red, um, blue.", expected: "Make it red, um, blue." },
+ {
+ cue: "uh no",
+ remove: true,
+ text: "Make it red, uh no, blue.",
+ expected: "Make it red, uh no, blue.",
+ },
+ {
+ cue: "uh no",
+ remove: true,
+ text: "Uh, Make it red, UH NO, blue. Uh nope.",
+ expected: "Make it red, UH NO, blue. nope.",
+ },
+ {
+ cue: " UM ",
+ remove: true,
+ text: "Uh, Make it red, um, blue.",
+ expected: "Make it red, um, blue.",
+ },
+ {
+ cue: undefined,
+ remove: true,
+ text: "Uh, Make it red, um, blue.",
+ expected: "Make it red, blue.",
+ },
+ {
+ cue: "um",
+ remove: false,
+ text: "Uh, Make it red, um, blue.",
+ expected: "Uh, Make it red, um, blue.",
+ },
+])(
+ "preserves only an explicit correction cue during batch and stream filler removal: $cue / $remove",
+ ({ cue, remove, text, expected }) =>
+ Effect.gen(function* () {
+ const speech = yield* SpeechService.SpeechService;
+ const options = {
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ speechLanguage: "en",
+ speechRemoveFillerWords: remove,
+ speechCustomWords: [{ term: "um", aliases: [] }],
+ ...(cue === undefined ? {} : { speechCorrectionWord: cue }),
+ };
+ native.transcribe.mockResolvedValueOnce({ text });
+ native.finish.mockResolvedValueOnce(text);
+ expect(yield* speech.transcribe(pcm(), options)).toBe(expected);
+ const stream = yield* speech.startStream(options);
+ expect(yield* stream.finish).toBe(expected);
+ }).pipe(Effect.scoped, Effect.provide(layer)),
+);
+
it.effect("holds model ownership until the stream scope closes and preserves silence", () =>
Effect.gen(function* () {
const speech = yield* SpeechService.SpeechService;
@@ -560,9 +720,13 @@ it.effect("replays streamed audio on CPU after an Auto-placed feed crashes, once
};
expect(yield* stream.feed(pcm())).toEqual({ revision: 5, text: recovered });
expect(loadNative.mock.calls.slice(-2).map((call) => call[3])).toEqual(["auto", "cpu"]);
- expect(native.begin.mock.calls.slice(-2)).toEqual([
- ["en", ["GitHub", "Bob"]],
- ["en", cpuSupportsVocabulary ? ["GitHub", "Bob"] : undefined],
+ expect(native.begin.mock.calls.slice(-2).map(([options]) => options)).toEqual([
+ { timestamps: "none", language: "en", vocabulary: ["GitHub", "Bob"] },
+ {
+ timestamps: "none",
+ language: "en",
+ ...(cpuSupportsVocabulary ? { vocabulary: ["GitHub", "Bob"] } : {}),
+ },
]);
expect(native.feed).toHaveBeenCalledTimes(feedCalls + 4);
expect(yield* stream.feed(pcm())).toEqual({ revision: 6, text: recovered });
@@ -583,7 +747,7 @@ it.effect.each([false, true])(
const cpu = {
...native,
backend: "CPU",
- begin: vi.fn(async (_language?: string) => {}),
+ begin: vi.fn(async (_options?: unknown) => {}),
feed: vi.fn(async (_pcm: Float32Array) => ({
revision: 1,
text: { committed: "", tentative: "" },
@@ -611,7 +775,11 @@ it.effect.each([false, true])(
expect(cpu.feed.mock.calls.map(([chunk]) => [...chunk])).toEqual(
chunks.map((chunk) => [...chunk]),
);
- expect(cpu.begin).toHaveBeenCalledWith("fr", undefined);
+ expect(cpu.begin).toHaveBeenCalledWith(
+ { timestamps: "none", language: "fr" },
+ true,
+ "test.gguf",
+ );
expect(gpu.dispose).toHaveBeenCalledOnce();
expect(cpu.dispose).not.toHaveBeenCalled();
const busy = yield* Effect.result(speech.selectModel("fallback-model"));
diff --git a/apps/server/src/speech/SpeechService.ts b/apps/server/src/speech/SpeechService.ts
index d1ad3b22a94..c1f762f2608 100644
--- a/apps/server/src/speech/SpeechService.ts
+++ b/apps/server/src/speech/SpeechService.ts
@@ -32,6 +32,7 @@ import {
type SpeechModel,
} from "./model.ts";
import { listNativeSpeechGpuDevices, loadNativeSpeechModel } from "./native.ts";
+import { SILERO_VAD } from "./silero.ts";
import { isSpeechPlatformSupported } from "./platform.ts";
import * as SpeechModelStore from "./SpeechModelStore.ts";
@@ -39,14 +40,13 @@ const SAMPLE_RATE = 16_000;
const MAX_SPEECH_DURATION_SECONDS = 5 * 60;
export const MAX_SPEECH_BYTES =
SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * MAX_SPEECH_DURATION_SECONDS;
-const MIN_CAPTURE_RMS = 0.0005;
export class SpeechInvalidAudioError extends Schema.TaggedError<SpeechInvalidAudioError>()(
"SpeechInvalidAudioError",
{ byteLength: Schema.Number, message: Schema.String },
) {}
-export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false): Float32Array {
+export function decodeSpeechPcm(pcmBytes: Uint8Array): Float32Array {
if (
pcmBytes.byteLength === 0 ||
pcmBytes.byteLength > MAX_SPEECH_BYTES ||
@@ -60,7 +60,6 @@ export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false):
// WebSocket frames can be pooled Node Buffers whose backing storage is larger
// than the frame. Copy the view so only the received bytes are decoded.
const pcm = new Float32Array(Uint8Array.from(pcmBytes).buffer);
- let energy = 0;
for (const sample of pcm) {
if (!Number.isFinite(sample) || sample < -1 || sample > 1) {
throw new SpeechInvalidAudioError({
@@ -68,11 +67,8 @@ export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false):
message: "audio contains an invalid PCM sample",
});
}
- energy += sample * sample;
}
- return !preserveSilence && Math.sqrt(energy / pcm.length) < MIN_CAPTURE_RMS
- ? new Float32Array()
- : pcm;
+ return pcm;
}
export class SpeechOperationError extends Schema.TaggedError<SpeechOperationError>()(
@@ -281,6 +277,23 @@ const make = Effect.gen(function* () {
const selectedModel = (settings: SettingsSnapshot): SpeechModel =>
getSpeechModel(settings.speechModelId) ?? getSpeechModel(DEFAULT_SPEECH_MODEL_ID)!;
+ let silenceRetryAt = 0;
+ const acquireSilenceModel = Effect.fn("SpeechService.acquireSilenceModel")(function* (
+ purpose: "installation" | "recording",
+ ) {
+ if (purpose === "recording" && now() < silenceRetryAt) return undefined;
+ const path = yield* store.download(SILERO_VAD).pipe(
+ Effect.timeout(purpose === "recording" ? "15 seconds" : "2 minutes"),
+ Effect.orElseSucceed(() => undefined),
+ );
+ if (path !== undefined) {
+ silenceRetryAt = 0;
+ return path;
+ }
+ silenceRetryAt = now() + 10 * 60_000;
+ yield* Effect.logWarning("Silero unavailable; using raw audio with bounded ASR processing.");
+ });
+
// Publish the reservation before forking: a cached transfer can finish before forkIn returns.
const downloadFiber = Effect.fn("SpeechService.downloadFiber")(function* (modelId: string) {
const definition = getSpeechModel(modelId);
@@ -306,6 +319,9 @@ const make = Effect.gen(function* () {
},
})
.pipe(
+ (modelDownload) =>
+ Effect.all([modelDownload, acquireSilenceModel("installation")], { concurrency: 2 }),
+ Effect.map(([modelPath]) => modelPath),
Effect.mapError((cause) => speechError("model download", cause)),
Effect.forkIn(scope),
);
@@ -442,6 +458,7 @@ const make = Effect.gen(function* () {
model: definition.name,
size: definition.size,
supportsStreaming: definition.supportsStreaming,
+ supportsBufferedStreaming: true,
supportsTranslation: definition.supportsTranslation,
languages: definition.languages,
supportsLanguageDetection: definition.supportsLanguageDetection,
@@ -512,7 +529,9 @@ const make = Effect.gen(function* () {
language: string | undefined,
) => {
return (
- options.speechRemoveFillerWords ? removeSpeechFillerWords(text, language) : text
+ options.speechRemoveFillerWords
+ ? removeSpeechFillerWords(text, language, options.speechCorrectionWord)
+ : text
).trim();
};
@@ -543,6 +562,15 @@ const make = Effect.gen(function* () {
),
);
+ const silenceModelPath = Effect.fn("SpeechService.silenceModelPath")(function* (
+ loaded: LoadedModel,
+ options: SpeechTranscriptionOptions,
+ ) {
+ if (!options.speechSilenceFiltering && (loaded.supportsStreaming || loaded.supportsLongForm))
+ return undefined;
+ return yield* acquireSilenceModel("recording");
+ });
+
const transcribe: SpeechService["Service"]["transcribe"] = Effect.fn("SpeechService.transcribe")(
function* (pcmBytes, options) {
const settings = yield* readSettings("transcription");
@@ -574,6 +602,7 @@ const make = Effect.gen(function* () {
),
catch: (cause) => speechError("model preparation", cause),
});
+ const vadPath = yield* silenceModelPath(loaded, options);
const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
const customWords = dictionary.map(({ term }) => term);
// The vocabulary capability guarantees plain transcription, not translation.
@@ -584,14 +613,19 @@ const make = Effect.gen(function* () {
try: (signal) => {
// Interruption kills the isolated process; the next request reloads the model.
signal.addEventListener("abort", () => dropModel(inference), { once: true });
- return inference.transcribe(pcm, {
- timestamps: "none",
- ...(language === "auto" ? {} : { language }),
- ...(translateToEnglish
- ? { task: "translate" as const, targetLanguage: "en" as const }
- : {}),
- ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}),
- });
+ return inference.transcribe(
+ pcm,
+ {
+ timestamps: "none",
+ ...(language === "auto" ? {} : { language }),
+ ...(translateToEnglish
+ ? { task: "translate" as const, targetLanguage: "en" as const }
+ : {}),
+ ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}),
+ },
+ options.speechSilenceFiltering,
+ vadPath,
+ );
},
catch: (cause) => {
dropModel(inference);
@@ -703,12 +737,28 @@ const make = Effect.gen(function* () {
loadModel(modelPath, definition, signal, settings.speechAcceleration),
);
loaded = streamModel;
+ const vadPath = yield* silenceModelPath(streamModel, options);
+ let translateToEnglish = false;
const beginStream = async () => {
- if (!streamModel.supportsStreaming)
- throw new Error("The selected model does not support streaming.");
+ translateToEnglish =
+ options.speechTranslateToEnglish &&
+ !streamModel.supportsStreaming &&
+ streamModel.supportsTranslation &&
+ language !== "en";
const vocabulary =
- streamModel.supportsVocabulary && customWords.length > 0 ? customWords : undefined;
- await streamModel.begin(language === "auto" ? undefined : language, vocabulary);
+ streamModel.supportsVocabulary && !translateToEnglish && customWords.length
+ ? customWords
+ : undefined;
+ await streamModel.begin(
+ {
+ timestamps: "none",
+ ...(language === "auto" ? {} : { language }),
+ ...(translateToEnglish ? { task: "translate", targetLanguage: "en" } : {}),
+ ...(vocabulary ? { vocabulary } : {}),
+ },
+ options.speechSilenceFiltering,
+ vadPath,
+ );
correct = makeSpeechTextCorrector(vocabulary ? [] : customWords, dictionary);
started = true;
};
@@ -768,7 +818,7 @@ const make = Effect.gen(function* () {
byteLength: bytes.byteLength,
message: "Speech stream audio limit exceeded.",
});
- const pcm = decodeSpeechPcm(bytes, true);
+ const pcm = decodeSpeechPcm(bytes);
byteLength += bytes.byteLength;
received.push(pcm);
let update: Awaited<ReturnType<LoadedModel["feed"]>> | undefined;
@@ -806,7 +856,7 @@ const make = Effect.gen(function* () {
return finalizeTranscript(
correct(text),
options,
- language === "auto" ? undefined : language,
+ translateToEnglish ? "en" : language === "auto" ? undefined : language,
);
}),
};
diff --git a/apps/server/src/speech/audio.test.ts b/apps/server/src/speech/audio.test.ts
new file mode 100644
index 00000000000..d049f8da51c
--- /dev/null
+++ b/apps/server/src/speech/audio.test.ts
@@ -0,0 +1,92 @@
+import { expect, it } from "vite-plus/test";
+import { createSpeechAudio } from "./audio.ts";
+
+const frame = 512;
+const make = (overrides = {}) =>
+ createSpeechAudio({
+ frameSamples: frame,
+ filterSilence: true,
+ nativeStreaming: false,
+ longForm: false,
+ maxAudioMs: 0,
+ ...overrides,
+ });
+const frames = (count: number) =>
+ Float32Array.from({ length: count * frame }, (_, i) => (Math.floor(i / frame) + 1) / 1000);
+const seconds = (n: number) => Math.floor((n * 16000) / frame);
+
+it("retains pre-roll, onset and offline/streaming tails without duplicating edges", () => {
+ for (const nativeStreaming of [false, true]) {
+ const audio = make({ nativeStreaming });
+ const pcm = frames(100);
+ const chunks: Float32Array[] = [];
+ for (let i = 0; i < 100; i++) {
+ audio.append(pcm.subarray(i * frame, (i + 1) * frame));
+ audio.score(i, [i >= 20 && i < 24 ? 0.9 : 0]);
+ chunks.push(...audio.take());
+ }
+ chunks.push(...audio.take(true));
+ const retained = chunks.flatMap((chunk) => [...chunk]);
+ expect(retained).toEqual([
+ ...pcm.subarray(6 * frame, (24 + (nativeStreaming ? 52 : 15)) * frame),
+ ]);
+ }
+});
+
+it("handles silence, short speech, a partial last frame and disabled filtering", () => {
+ const silence = make();
+ silence.append(new Float32Array(frame * 4 + 7));
+ silence.score(0, [0, 0, 0, 0, 0]);
+ expect(silence.take(true)).toEqual([]);
+ const short = make();
+ short.append(new Float32Array([0.2, 0.1]));
+ short.score(0, [0.8]);
+ expect([...short.take(true)[0]!]).toEqual([...new Float32Array([0.2, 0.1])]);
+ const raw = make({ filterSilence: false });
+ const pcm = frames(50).subarray(0, 50 * frame - 3);
+ raw.append(pcm);
+ raw.score(0, new Array(50).fill(0));
+ expect(raw.take(true)[0]).toEqual(pcm);
+});
+
+it("plans at pauses while recording and preserves ordered audio and final tails", () => {
+ const audio = make({ filterSilence: false });
+ const pcm = frames(seconds(100));
+ const chunks: Float32Array[] = [];
+ for (let start = 0; start < pcm.length; start += frame * 7) {
+ const piece = pcm.subarray(start, start + frame * 7);
+ audio.append(piece);
+ audio.score(
+ start / frame,
+ Array.from({ length: piece.length / frame }, (_, i) => {
+ const t = ((start / frame + i) * frame) / 16000;
+ return (t >= 28 && t < 29) || (t >= 55 && t < 56) ? 0 : 1;
+ }),
+ );
+ chunks.push(...audio.take());
+ }
+ expect(chunks.length).toBeGreaterThan(0);
+ expect(chunks[0]!.length / 16000).toBeCloseTo(28.5, 1);
+ chunks.push(...audio.take(true));
+ expect(Float32Array.from(chunks.flatMap((chunk) => [...chunk]))).toEqual(pcm);
+ expect(chunks.every((chunk) => chunk.length <= 40 * 16000)).toBe(true);
+ expect(chunks.at(-1)!.length).toBeGreaterThanOrEqual(5 * 16000);
+});
+
+it("respects smaller engine limits, hard cuts, and native/internal-long-form eligibility", () => {
+ for (const config of [{}, { maxAudioMs: 4000 }, { nativeStreaming: true }, { longForm: true }]) {
+ const audio = make({ filterSilence: false, ...config });
+ const pcm = frames(seconds(84));
+ audio.append(pcm);
+ audio.score(0, new Array(pcm.length / frame).fill(1));
+ const live = audio.take();
+ const chunks = [...live, ...audio.take(true)];
+ expect(chunks.reduce((n, chunk) => n + chunk.length, 0)).toBe(pcm.length);
+ if ("nativeStreaming" in config || "longForm" in config) expect(chunks).toHaveLength(1);
+ else
+ expect(
+ chunks.every((chunk) => chunk.length <= ("maxAudioMs" in config ? 32000 : 640000)),
+ ).toBe(true);
+ if ("longForm" in config) expect(live).toEqual([]);
+ }
+});
diff --git a/apps/server/src/speech/audio.ts b/apps/server/src/speech/audio.ts
new file mode 100644
index 00000000000..62f31f182c8
--- /dev/null
+++ b/apps/server/src/speech/audio.ts
@@ -0,0 +1,174 @@
+// Self-contained because the native worker embeds this factory in Node, Electron and SEA.
+export function createSpeechAudio(config: {
+ frameSamples: number;
+ filterSilence: boolean;
+ nativeStreaming: boolean;
+ longForm: boolean;
+ maxAudioMs: number;
+}) {
+ const rate = 16_000;
+ const frame = config.frameSamples;
+ const prefill = Math.ceil((450 * rate) / 1000 / frame);
+ const onset = Math.ceil((60 * rate) / 1000 / frame);
+ const tail = Math.ceil(((config.nativeStreaming ? 1650 : 450) * rate) / 1000 / frame);
+ const eligible = !config.nativeStreaming && !config.longForm;
+ // Preserve the existing output-limit headroom and bounded offline retries.
+ const maximum = Math.min(
+ 40 * rate,
+ config.maxAudioMs > 0 ? Math.max(1, Math.floor((config.maxAudioMs * rate) / 2000)) : 40 * rate,
+ );
+ const minimum = Math.min(5 * rate, Math.floor(maximum / 4));
+ const target = Math.min(30 * rate, maximum * 0.75);
+ const blocks: { start: number; pcm: Float32Array }[] = [];
+ const probs: number[] = [];
+ const keep: boolean[] = [];
+ let length = 0;
+ let start = 0;
+ let streamAt = 0;
+ let inSpeech = false;
+ let onsetCount = 0;
+ let hangover = 0;
+
+ const raw = (from: number, to: number) => {
+ const output = new Float32Array(to - from);
+ for (const block of blocks) {
+ const left = Math.max(from, block.start);
+ const right = Math.min(to, block.start + block.pcm.length);
+ if (left < right)
+ output.set(block.pcm.subarray(left - block.start, right - block.start), left - from);
+ }
+ return output;
+ };
+ const read = (from: number, to: number) => {
+ const pcm = raw(from, to);
+ if (!config.filterSilence) return pcm;
+ let count = 0;
+ for (let at = from; at < to;) {
+ const end = Math.min(to, (Math.floor(at / frame) + 1) * frame);
+ if (keep[Math.floor(at / frame)]) {
+ pcm.copyWithin(count, at - from, end - from);
+ count += end - at;
+ }
+ at = end;
+ }
+ return pcm.subarray(0, count);
+ };
+ // Silero timestamp hysteresis is separate from the capture filter's 0.3 threshold.
+ const pauses = (samples: number, final: boolean) => {
+ const segments: { start: number; end: number }[] = [];
+ let speechStart: number | undefined;
+ let silenceStart: number | undefined;
+ for (let i = 0; i < probs.length; i++) {
+ const at = i * frame;
+ if (probs[i]! >= 0.5) {
+ silenceStart = undefined;
+ speechStart ??= at;
+ } else if (probs[i]! < 0.35 && speechStart !== undefined) {
+ silenceStart ??= at;
+ if (at - silenceStart >= 0.1 * rate) {
+ if (silenceStart - speechStart > 0.25 * rate)
+ segments.push({ start: speechStart, end: silenceStart });
+ speechStart = silenceStart = undefined;
+ }
+ }
+ }
+ if (speechStart !== undefined && samples - speechStart > 0.25 * rate)
+ segments.push({ start: speechStart, end: samples });
+ const candidates: { cut: number; gap: number }[] = [];
+ let end = 0;
+ for (const segment of segments) {
+ if (segment.start > end)
+ candidates.push({
+ cut: Math.floor((end + segment.start) / 2),
+ gap: (segment.start - end) / rate,
+ });
+ end = segment.end;
+ }
+ // An open pause's midpoint can still move, so only final audio uses it.
+ if (final && samples > end)
+ candidates.push({ cut: Math.floor((end + samples) / 2), gap: (samples - end) / rate });
+ return candidates;
+ };
+
+ return {
+ append(pcm: Float32Array) {
+ blocks.push({ start: length, pcm });
+ length += pcm.length;
+ },
+ score(firstFrame: number, scores: readonly number[]) {
+ if (firstFrame !== probs.length || scores.some((p) => !Number.isFinite(p) || p < 0 || p > 1))
+ throw new Error("Invalid Silero frame sequence.");
+ for (const probability of scores) {
+ const i = probs.length;
+ probs.push(probability);
+ keep.push(false);
+ const voiced = probability > 0.3;
+ if (!inSpeech && voiced) {
+ if (++onsetCount >= onset) {
+ inSpeech = true;
+ hangover = tail;
+ onsetCount = 0;
+ for (let j = Math.max(0, i - prefill); j <= i; j++) keep[j] = true;
+ }
+ } else if (inSpeech && voiced) {
+ hangover = tail;
+ keep[i] = true;
+ } else if (inSpeech && hangover > 0) {
+ hangover--;
+ keep[i] = true;
+ } else {
+ inSpeech = false;
+ onsetCount = 0;
+ }
+ }
+ },
+ audio: () => raw(0, length),
+ samples: () => length,
+ take(final = false): Float32Array[] {
+ const samples = Math.min(length, probs.length * frame);
+ // Preserve a very short utterance or syllable stopped before onset confirms.
+ if (final && onsetCount > 0)
+ for (let i = Math.max(0, probs.length - 1 - prefill); i < probs.length; i++) keep[i] = true;
+ if (config.nativeStreaming) {
+ const output = read(streamAt, samples);
+ let lastKept = streamAt;
+ for (let i = Math.floor(streamAt / frame); i < probs.length; i++)
+ if (!config.filterSilence || keep[i]) lastKept = Math.min(samples, (i + 1) * frame);
+ streamAt = Math.max(lastKept, Math.max(0, samples - prefill * frame));
+ return output.length ? [output] : [];
+ }
+ const chunks: Float32Array[] = [];
+ const lookahead = Math.max(minimum, (prefill + onset) * frame);
+ if (eligible && samples - start > maximum + (final ? 0 : lookahead)) {
+ const candidates = pauses(samples, final);
+ while (samples - start > maximum + (final ? 0 : lookahead)) {
+ let best: { cut: number; score: number } | undefined;
+ let fallback: typeof best;
+ for (const candidate of candidates) {
+ const position = candidate.cut - start;
+ if (position < minimum || position > maximum) continue;
+ const offTarget = (position - target) / rate;
+ const score =
+ Math.min(candidate.gap, 1) - Math.abs(offTarget) * (offTarget > 0 ? 0.1 : 0.05);
+ if (!fallback || score > fallback.score) fallback = { cut: candidate.cut, score };
+ if (samples - candidate.cut >= minimum && (!best || score > best.score))
+ best = { cut: candidate.cut, score };
+ }
+ const hard =
+ start +
+ (samples - start < maximum + minimum ? Math.floor((samples - start) / 2) : maximum);
+ const cut = (best ?? fallback)?.cut ?? hard;
+ const chunk = read(start, cut);
+ if (chunk.length) chunks.push(chunk);
+ start = cut;
+ }
+ }
+ if (final && samples > start) {
+ const chunk = read(start, samples);
+ if (chunk.length) chunks.push(chunk);
+ start = samples;
+ }
+ return chunks;
+ },
+ };
+}
diff --git a/apps/server/src/speech/fillerWords.ts b/apps/server/src/speech/fillerWords.ts
index 0181db1bf91..5061341833c 100644
--- a/apps/server/src/speech/fillerWords.ts
+++ b/apps/server/src/speech/fillerWords.ts
@@ -22,19 +22,21 @@ const LANGUAGE_FILLER_WORDS: Readonly<Record<string, readonly string[]>> = {
const escapeRegex = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-export function removeSpeechFillerWords(text: string, language?: string): string {
+export function removeSpeechFillerWords(
+ text: string,
+ language?: string,
+ correctionCue?: string,
+): string {
const baseLanguage = language?.split(/[-_]/, 1)[0]?.toLowerCase();
const words = [
...UNIVERSAL_FILLER_WORDS,
...(baseLanguage ? (LANGUAGE_FILLER_WORDS[baseLanguage] ?? []) : []),
];
- let filtered = text;
- for (const word of words) {
- const pattern = new RegExp(
- `(?<![\\p{L}\\p{N}_])${escapeRegex(word)}(?![\\p{L}\\p{N}_])[,.]?`,
- "giu",
- );
- filtered = filtered.replace(pattern, "");
- }
+ const cuePattern = correctionCue?.trim().split(/\s+/).map(escapeRegex).join("\\s+");
+ const pattern = new RegExp(
+ `(?<![\\p{L}\\p{N}_])(?:(${cuePattern || "(?!)"})(?![\\p{L}\\p{N}_])|(?:${words.map(escapeRegex).join("|")})(?![\\p{L}\\p{N}_])[,.]?)`,
+ "giu",
+ );
+ const filtered = text.replace(pattern, (_match, cue: string | undefined) => cue ?? "");
return filtered.replace(/\s{2,}/g, " ").trim();
}
diff --git a/apps/server/src/speech/native.test.ts b/apps/server/src/speech/native.test.ts
index da92597d06e..97aa1b56602 100644
--- a/apps/server/src/speech/native.test.ts
+++ b/apps/server/src/speech/native.test.ts
@@ -39,7 +39,11 @@ it("returns transcription from an isolated process and disposes it", async () =>
);
try {
expect(
- await model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }),
+ await model.transcribe(
+ new Float32Array([0.25]),
+ { timestamps: "none", language: "en" },
+ false,
+ ),
).toEqual({ text: "0.25" });
} finally {
await model.dispose();
@@ -60,7 +64,7 @@ it("probes generic vocabulary support and delivers ordered terms to offline infe
language: "en",
...(supported ? { vocabulary: ["T3 Code", "GitHub"] } : {}),
};
- const result = await model.transcribe(new Float32Array([0.25]), options);
+ const result = await model.transcribe(new Float32Array([0.25]), options, false);
expect(JSON.parse(result.text)).toEqual(options);
} finally {
await model.dispose();
@@ -76,7 +80,7 @@ it("ignores IPC messages that do not belong to the speech protocol", async () =>
);
try {
await expect(
- model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }),
+ model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }, false),
).resolves.toEqual({ text: "hello" });
} finally {
await model.dispose();
@@ -90,10 +94,14 @@ it("terminates hung native inference when the service shuts down", async () => {
controller.signal,
fixture("async () => { while (true) {} }"),
);
- const pending = model.transcribe(new Float32Array([0.25]), {
- timestamps: "none",
- language: "en",
- });
+ const pending = model.transcribe(
+ new Float32Array([0.25]),
+ {
+ timestamps: "none",
+ language: "en",
+ },
+ false,
+ );
const rejected = expect(pending).rejects.toThrow();
controller.abort();
await model.dispose();
@@ -111,17 +119,22 @@ it("returns streaming previews and final text from an isolated process", async (
);
try {
expect(model.supportsStreaming).toBe(true);
- await model.begin();
- expect(await model.feed(new Float32Array([0.25]))).toEqual({
+ await model.begin({ timestamps: "none" }, false);
+ expect(await model.feed(new Float32Array(512).fill(0.25))).toEqual({
revision: 1,
text: { committed: "hello ", tentative: "world" },
});
expect(await model.finish()).toBe("hello world");
- await model.begin("fr");
+ await model.begin({ timestamps: "none", language: "fr" }, false);
await model.reset();
- await model.begin("fr");
+ await model.begin({ timestamps: "none", language: "fr" }, false);
+ await model.feed(new Float32Array(512));
expect(await model.finish()).toBe("fr");
- await model.begin("fr", ["T3 Code", "GitHub"]);
+ await model.begin(
+ { timestamps: "none", language: "fr", vocabulary: ["T3 Code", "GitHub"] },
+ false,
+ );
+ await model.feed(new Float32Array(512).fill(0.25));
expect(await model.finish()).toBe("T3 Code / GitHub");
} finally {
await model.dispose();
@@ -135,9 +148,170 @@ it("kills a hung streaming feed without waiting for native cleanup", async () =>
controller.signal,
streamingFixture("async () => { while (true) {} }"),
);
- await model.begin();
- const rejected = expect(model.feed(new Float32Array([0.25]))).rejects.toThrow();
+ await model.begin({ timestamps: "none" }, false);
+ const rejected = expect(model.feed(new Float32Array(512).fill(0.25))).rejects.toThrow();
controller.abort();
await model.dispose();
await rejected;
});
+const vadFixture = ({
+ streaming = false,
+ failVad = false,
+ failVadLoad = false,
+ failAsr = false,
+ maxAudioMs = 4000,
+} = {}) =>
+ `data:text/javascript,${encodeURIComponent(`
+ const summarize = pcm => String(pcm.length) + ":" + pcm[0].toFixed(2) + ":" + pcm[pcm.length - 1].toFixed(2);
+ export const TranscribeModel = { load: async (path, options) => {
+ if (path === "vad.gguf") {
+ if (${failVadLoad}) throw new Error("VAD unavailable");
+ if (options.backend !== "cpu") throw new Error("VAD must use CPU");
+ return { vadInfo: { sampleRate: 16000, frameSamples: 512 }, dispose() {},
+ createVadSession: ({ nThreads }) => {
+ if (nThreads !== 1) throw new Error("VAD thread count");
+ let pending = []; let at = 0;
+ const score = final => {
+ const firstFrame = at;
+ const probs = [];
+ while (pending.length >= 512 || final && pending.length) {
+ probs.push(pending[0] > 0.5 ? 0.9 : 0);
+ pending.splice(0, 512); at++;
+ }
+ return { firstFrame, probs };
+ };
+ return { dispose() {}, streamReset: async () => { pending = []; at = 0; },
+ streamFeed: async pcm => {
+ if (${failVad} && pcm[0] < 0) throw new Error("VAD fixture failed");
+ pending.push(...pcm); return score(false);
+ },
+ streamFlush: async () => score(true),
+ };
+ },
+ };
+ }
+ return { backend: "CPU", supports: () => false,
+ capabilities: { supportsStreaming: ${streaming}, maxAudioMs: ${maxAudioMs} },
+ transcribe: async pcm => { if (${failAsr}) throw new Error("ASR fixture failed"); return { text: summarize(pcm) }; },
+ createSession: () => {
+ let all = [];
+ return { dispose() {}, stream: async () => ({
+ feed: async pcm => { all.push(...pcm); return { committedChanged: true }; },
+ finalize: async () => {}, reset() { all = []; },
+ get text() { const full = summarize(all); return { committed: full, tentative: "", full }; },
+ }) };
+ },
+ };
+ }};
+ `)}`;
+
+it("filters silence and preserves partial speech through the production worker and resets between recordings", async () => {
+ for (const streaming of [false, true]) {
+ const model = await loadNativeSpeechModel(
+ "asr.gguf",
+ new AbortController().signal,
+ vadFixture({ streaming }),
+ "cpu",
+ );
+ try {
+ expect(
+ await model.transcribe(new Float32Array(2000), { timestamps: "none" }, true, "vad.gguf"),
+ ).toEqual({ text: "" });
+ expect(
+ await model.transcribe(
+ new Float32Array([0.8, 0.6]),
+ { timestamps: "none" },
+ true,
+ "vad.gguf",
+ ),
+ ).toEqual({ text: "2:0.80:0.60" });
+ expect(
+ await model.transcribe(new Float32Array(2000), { timestamps: "none" }, false, "vad.gguf"),
+ ).toEqual({ text: "2000:0.00:0.00" });
+ for (const cancel of [true, false]) {
+ await model.begin({ timestamps: "none" }, true, "vad.gguf");
+ await model.feed(new Float32Array(600));
+ expect(await model.finish()).toBe("");
+ await model.begin({ timestamps: "none" }, true, "vad.gguf");
+ await model.feed(new Float32Array(511).fill(0.8));
+ if (cancel) await model.reset();
+ else expect(await model.finish()).toBe("511:0.80:0.80");
+ }
+ } finally {
+ await model.dispose();
+ }
+ }
+});
+
+it("decodes complete stable chunks before Stop and returns the ordered final tail", async () => {
+ const model = await loadNativeSpeechModel(
+ "asr.gguf",
+ new AbortController().signal,
+ vadFixture(),
+ "cpu",
+ );
+ try {
+ await model.begin({ timestamps: "none" }, false, "vad.gguf");
+ await model.feed(new Float32Array(16000).fill(0.1));
+ await model.feed(new Float32Array(16000).fill(0.2));
+ const update = await model.feed(new Float32Array(16000).fill(0.3));
+ expect(update.text).toEqual({ committed: "32000:0.10:0.20", tentative: "" });
+ await model.feed(new Float32Array([0.4, 0.5]));
+ expect(await model.finish()).toBe("32000:0.10:0.20 16002:0.30:0.50");
+ } finally {
+ await model.dispose();
+ }
+});
+
+it("preserves withheld audio on VAD failure while keeping ASR failures distinct", async () => {
+ for (const streaming of [false, true]) {
+ const model = await loadNativeSpeechModel(
+ "asr.gguf",
+ new AbortController().signal,
+ vadFixture({ streaming, failVad: true }),
+ "cpu",
+ );
+ try {
+ await model.begin({ timestamps: "none" }, true, "vad.gguf");
+ await model.feed(new Float32Array(512).fill(0.1));
+ await model.feed(new Float32Array(512).fill(-0.2));
+ expect(await model.finish()).toBe("1024:0.10:-0.20");
+ await model.begin({ timestamps: "none" }, true, "vad.gguf");
+ await model.feed(new Float32Array(512));
+ expect(await model.finish()).toBe("");
+ } finally {
+ await model.dispose();
+ }
+ }
+ const unavailable = await loadNativeSpeechModel(
+ "asr.gguf",
+ new AbortController().signal,
+ vadFixture({ failVadLoad: true }),
+ "cpu",
+ );
+ try {
+ expect(
+ await unavailable.transcribe(
+ new Float32Array(2000).fill(0.1),
+ { timestamps: "none" },
+ true,
+ "vad.gguf",
+ ),
+ ).toEqual({ text: "2000:0.10:0.10" });
+ } finally {
+ await unavailable.dispose();
+ }
+ const model = await loadNativeSpeechModel(
+ "asr.gguf",
+ new AbortController().signal,
+ vadFixture({ failAsr: true }),
+ "cpu",
+ );
+ try {
+ await expect(
+ model.transcribe(new Float32Array(512).fill(0.8), { timestamps: "none" }, true, "vad.gguf"),
+ ).rejects.toThrow("ASR fixture failed");
+ } finally {
+ await model.dispose();
+ }
+});
diff --git a/apps/server/src/speech/native.ts b/apps/server/src/speech/native.ts
index 88809ab9766..fae308333b2 100644
--- a/apps/server/src/speech/native.ts
+++ b/apps/server/src/speech/native.ts
@@ -2,16 +2,142 @@
import * as NodeChildProcess from "node:child_process";
import * as NodeSea from "node:sea";
import type { SpeechStreamText } from "@t3tools/contracts";
+import { createSpeechAudio } from "./audio.ts";
// Inline the small child entry so the packaged CLI does not need a separate worker artifact.
const entry = `
+const createSpeechAudio = ${createSpeechAudio.toString()};
+let speech;
let model;
+let vadModel;
+let vad;
let session;
let stream;
+let audio;
+let options;
+let nativeStreaming;
+let useVad;
+let scored;
+let revision;
+let texts;
+let heardAudio;
+const warnVad = () => console.warn("Silero silence filtering unavailable; preserving raw recording and using bounded ASR chunks.");
+const resetAsr = () => {
+ if (stream) stream.reset();
+ if (session) session.dispose();
+ stream = session = undefined;
+};
+const newAudio = (filterSilence) => createSpeechAudio({
+ frameSamples: 512, filterSilence, nativeStreaming: model.capabilities.supportsStreaming,
+ longForm: model.supports("long_form"), maxAudioMs: model.capabilities.maxAudioMs,
+});
+const openAsr = async () => {
+ if (nativeStreaming) {
+ session = model.createSession();
+ stream = await session.stream(options);
+ }
+};
+const begin = async (message, live) => {
+ resetAsr();
+ options = message.options;
+ nativeStreaming = live && model.capabilities.supportsStreaming;
+ texts = [];
+ scored = revision = 0;
+ heardAudio = false;
+ useVad = Boolean(message.vadPath) && (message.filterSilence ||
+ (!model.capabilities.supportsStreaming && !model.supports("long_form")));
+ if (useVad) {
+ try {
+ if (!vadModel) {
+ vadModel = await speech.TranscribeModel.load(message.vadPath, { backend: "cpu" });
+ if (vadModel.vadInfo.sampleRate !== 16000 || vadModel.vadInfo.frameSamples !== 512)
+ throw new Error("Unsupported Silero model frames.");
+ vad = vadModel.createVadSession({ nThreads: 1 });
+ }
+ await vad.streamReset();
+ } catch {
+ vad?.dispose();
+ vadModel?.dispose();
+ vad = vadModel = undefined;
+ useVad = false;
+ warnVad();
+ }
+ }
+ audio = newAudio(useVad && message.filterSilence);
+ await openAsr();
+};
+const transcribe = async (pcm) => {
+ try {
+ const result = await model.transcribe(pcm, options);
+ texts.push(result.text);
+ } catch (error) {
+ if ((error?.constructor?.name !== "OutputTruncated" &&
+ error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error;
+ const middle = Math.floor(pcm.length / 2);
+ await transcribe(pcm.subarray(0, middle));
+ await transcribe(pcm.subarray(middle));
+ }
+};
+const decode = async (final) => {
+ let preview = null;
+ for (const pcm of audio.take(final)) {
+ if (!pcm.length) continue;
+ heardAudio = true;
+ if (nativeStreaming) {
+ const update = await stream.feed(pcm);
+ if (update.committedChanged || update.tentativeChanged) {
+ const text = stream.text;
+ preview = { committed: text.committed, tentative: text.tentative };
+ }
+ } else {
+ await transcribe(pcm);
+ preview = { committed: texts.join(" ").trim(), tentative: "" };
+ }
+ }
+ return preview;
+};
+const feed = async (pcm, final = false) => {
+ if (pcm) audio.append(pcm);
+ try {
+ if (useVad) {
+ if (pcm) {
+ const result = await vad.streamFeed(pcm);
+ audio.score(result.firstFrame, result.probs);
+ if (result.firstFrame + result.probs.length !== Math.floor(audio.samples() / 512))
+ throw new Error("Incomplete Silero frames.");
+ }
+ if (final) {
+ const result = await vad.streamFlush();
+ audio.score(result.firstFrame, result.probs);
+ if (result.firstFrame + result.probs.length !== Math.ceil(audio.samples() / 512))
+ throw new Error("Incomplete Silero tail.");
+ }
+ } else {
+ const count = final ? Math.ceil(audio.samples() / 512) : Math.floor(audio.samples() / 512);
+ audio.score(scored, new Array(count - scored).fill(1));
+ scored = count;
+ }
+ } catch {
+ // Replay on optional VAD failure, including audio it had withheld; ASR errors stay separate.
+ const raw = audio.audio();
+ useVad = false;
+ warnVad();
+ resetAsr();
+ texts = [];
+ heardAudio = false;
+ audio = newAudio(false);
+ audio.append(raw);
+ scored = final ? Math.ceil(raw.length / 512) : Math.floor(raw.length / 512);
+ audio.score(0, new Array(scored).fill(1));
+ await openAsr();
+ }
+ return await decode(final);
+};
process.on("message", async (message) => {
try {
if (message.kind === "load") {
- const { TranscribeModel, getAvailableBackends } = await loadSpeechModule(message.moduleUrl);
+ speech = await loadSpeechModule(message.moduleUrl);
+ const { TranscribeModel, getAvailableBackends } = speech;
const device = message.acceleration.startsWith("gpu:")
? getAvailableBackends().find((item) =>
(item.deviceType === "gpu" || item.deviceType === "igpu") &&
@@ -19,80 +145,40 @@ process.on("message", async (message) => {
: undefined;
if (message.acceleration.startsWith("gpu:") && !device)
throw new Error("The selected speech GPU is unavailable.");
- model = await TranscribeModel.load(message.path, device
- ? { device }
- : { backend: message.acceleration });
+ model = await TranscribeModel.load(message.path, device ? { device } : { backend: message.acceleration });
process.send({ type: "t3-speech-reply", ok: true, backend: model.backend,
supportsStreaming: model.capabilities.supportsStreaming,
+ supportsLongForm: model.supports("long_form"),
supportsTranslation: model.capabilities.supportsTranslate,
supportsVocabulary: model.supports("vocabulary") });
} else if (message.kind === "begin") {
- session = model.createSession();
- stream = await session.stream({ timestamps: "none", language: message.language,
- vocabulary: message.vocabulary });
+ await begin(message, true);
process.send({ type: "t3-speech-reply", ok: true });
} else if (message.kind === "feed") {
- const update = await stream.feed(message.pcm);
- const text = update.committedChanged || update.tentativeChanged ? stream.text : null;
- process.send({ type: "t3-speech-reply", ok: true, revision: update.revision,
- preview: text ? { committed: text.committed, tentative: text.tentative } : null });
+ const preview = await feed(message.pcm);
+ process.send({ type: "t3-speech-reply", ok: true, revision: ++revision, preview });
} else if (message.kind === "finish") {
- await stream.finalize();
- const text = stream.text.full;
- stream.reset();
- session.dispose();
- stream = session = undefined;
+ await feed(undefined, true);
+ if (nativeStreaming && heardAudio) await stream.finalize();
+ const text = nativeStreaming ? (heardAudio ? stream.text.full : "") : texts.join(" ").trim();
+ resetAsr();
+ audio = undefined;
process.send({ type: "t3-speech-reply", ok: true, text });
} else if (message.kind === "reset") {
- stream.reset();
- session.dispose();
- stream = session = undefined;
+ resetAsr();
+ if (vad) await vad.streamReset();
+ audio = undefined;
process.send({ type: "t3-speech-reply", ok: true });
} else if (message.kind === "transcribe") {
- // Leave unbounded models alone. Use headroom for output-bound models:
- // maxAudioMs is an estimate, so dense speech may still need smaller pieces.
- const limit = model.capabilities.maxAudioMs;
- const chunkSamples = limit > 0 ? Math.max(4000, Math.floor(limit * 16 / 2)) : message.pcm.length;
- const texts = [];
- const transcribe = async (pcm) => {
- try {
- const result = await model.transcribe(pcm, message.options);
- texts.push(result.text);
- } catch (error) {
- if ((error?.constructor?.name !== "OutputTruncated" &&
- error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error;
- const middle = Math.floor(pcm.length / 2);
- await transcribe(pcm.subarray(0, middle));
- await transcribe(pcm.subarray(middle));
- }
- };
- let offset = 0;
- while (offset < message.pcm.length) {
- let end = Math.min(offset + chunkSamples, message.pcm.length);
- if (end < message.pcm.length) {
- // Prefer the quietest 20 ms frame in the final quarter of the window.
- let quietest = Infinity;
- const windowEnd = end;
- for (let start = offset + Math.floor(chunkSamples * 0.75); start + 320 <= windowEnd; start += 320) {
- let energy = 0;
- for (let i = start; i < start + 320; i++) energy += message.pcm[i] ** 2;
- if (energy < quietest) {
- quietest = energy;
- end = start + 160;
- }
- }
- }
- await transcribe(message.pcm.subarray(offset, end));
- offset = end;
- }
- process.send({ type: "t3-speech-reply", ok: true, text: texts.join(" ").trim() });
+ await begin(message, false);
+ await feed(message.pcm, true);
+ const text = texts.join(" ").trim();
+ audio = undefined;
+ process.send({ type: "t3-speech-reply", ok: true, text });
}
} catch (error) {
- process.send({
- type: "t3-speech-reply",
- ok: false,
- error: error instanceof Error ? error.stack ?? error.message : String(error),
- });
+ process.send({ type: "t3-speech-reply", ok: false,
+ error: error instanceof Error ? error.stack ?? error.message : String(error) });
}
});
process.on("disconnect", () => process.exit(0));
@@ -104,6 +190,7 @@ type Reply = {
readonly text?: string;
readonly error?: string;
readonly supportsStreaming?: boolean;
+ readonly supportsLongForm?: boolean;
readonly supportsTranslation?: boolean;
readonly supportsVocabulary?: boolean;
readonly backend?: string;
@@ -305,10 +392,11 @@ async function loadNativeSpeechModelOnce(
return {
backend: loaded.backend ?? "unknown",
supportsStreaming: loaded.supportsStreaming === true,
+ supportsLongForm: loaded.supportsLongForm === true,
supportsTranslation: loaded.supportsTranslation === true,
supportsVocabulary: loaded.supportsVocabulary === true,
- begin: async (language?: string, vocabulary?: readonly string[]) => {
- await send({ kind: "begin", language, vocabulary });
+ begin: async (options: NativeSpeechOptions, filterSilence: boolean, vadPath?: string) => {
+ await send({ kind: "begin", options, filterSilence, vadPath });
},
feed: async (pcm: Float32Array) => {
const reply = await send({ kind: "feed", pcm });
@@ -343,18 +431,22 @@ async function loadNativeSpeechModelOnce(
},
transcribe: async (
pcm: Float32Array,
- options: {
- readonly timestamps: "none";
- readonly task?: "translate";
- readonly targetLanguage?: "en";
- readonly language?: string;
- readonly vocabulary?: readonly string[];
- },
+ options: NativeSpeechOptions,
+ filterSilence: boolean,
+ vadPath?: string,
) => {
- const result = await send({ kind: "transcribe", pcm, options });
+ const result = await send({ kind: "transcribe", pcm, options, filterSilence, vadPath });
if (typeof result.text !== "string") throw new Error("Invalid speech process response.");
return { text: result.text };
},
dispose,
};
}
+
+type NativeSpeechOptions = {
+ readonly timestamps: "none";
+ readonly task?: "translate";
+ readonly targetLanguage?: "en";
+ readonly language?: string;
+ readonly vocabulary?: readonly string[];
+};
diff --git a/apps/server/src/speech/silero.ts b/apps/server/src/speech/silero.ts
new file mode 100644
index 00000000000..dceddb71dde
--- /dev/null
+++ b/apps/server/src/speech/silero.ts
@@ -0,0 +1,8 @@
+// MIT-licensed internal helper; acquired and verified by the existing model store.
+export const SILERO_VAD = {
+ id: "handy-computer/silero-vad-v6.2-gguf",
+ revision: "97bbd75179f535e4708bccdf61e9278b8b60dfc7",
+ filename: "silero-vad-v6.2-F32.gguf",
+ size: 1_241_056,
+ sha256: "0d871a800c420fdd5d1b17a3aea4296d1c39608f0ff4a9295eb76457eb1c88e6",
+};
diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx
index 4408bee38ad..8e418324448 100644
--- a/apps/web/src/components/ChatView.tsx
+++ b/apps/web/src/components/ChatView.tsx
@@ -3245,6 +3245,7 @@ export default function ChatView(props: ChatViewProps) {
: undefined,
}}
files={{
+ projectId: activeProject?.id,
projectName: activeProject?.title ?? null,
workspaceRoot: activeWorkspaceRoot,
availableEditors,
@@ -3694,6 +3695,7 @@ export default function ChatView(props: ChatViewProps) {
}
activeThreadModelSelection={activeThread?.modelSelection}
activeProjectId={activeProject?.id ?? null}
+ projectName={activeProject?.title}
contextWindowEntries={serverVisibleTurnItems}
contextWindowLiveUsage={activeThreadLiveTokenUsage}
contextWindowProviderThread={activeProviderThread}
diff --git a/apps/web/src/components/DiffPanel.browser.test.tsx b/apps/web/src/components/DiffPanel.browser.test.tsx
index 3157c3e5da2..fb2fc81bae8 100644
--- a/apps/web/src/components/DiffPanel.browser.test.tsx
+++ b/apps/web/src/components/DiffPanel.browser.test.tsx
@@ -84,6 +84,7 @@ vi.mock("~/state/use-atom-command", () => ({ useAtomCommand: () => async () => {
vi.mock("~/editorPreferences", () => ({ useOpenInPreferredEditor: () => async () => {} }));
vi.mock("~/fileContextMenu", () => ({ useFileContextMenuHandler: () => () => {} }));
vi.mock("~/browser/useOpenLink", () => ({ useOpenLink: () => async () => {} }));
+vi.mock("./diffs/DiffCommentAnnotation", () => ({ DiffCommentAnnotation: () => null }));
vi.mock("./diffs/DiffFileTree", () => ({ DiffFileTree: () => null }));
vi.mock("./DiffFilePathCopyButton", () => ({ DiffFilePathCopyButton: () => null }));
@@ -250,7 +251,7 @@ it.each([false, true])(
const toggle = screen.getByRole("button", { name: `Expand ${path}`, exact: true });
await expect.element(toggle).toHaveAttribute("aria-expanded", "false");
await userEvent.click(toggle);
- act(() =>
+ await act(() =>
context.registry.set(
context.patch,
AsyncResult.success({
diff --git a/apps/web/src/components/DiffPanel.tsx b/apps/web/src/components/DiffPanel.tsx
index f40760f847e..e7e6d614ef7 100644
--- a/apps/web/src/components/DiffPanel.tsx
+++ b/apps/web/src/components/DiffPanel.tsx
@@ -1425,6 +1425,8 @@ export default function DiffPanel({
>
<AnnotatableCodeView
key={collapseScopeKey ?? reviewSectionId}
+ environmentId={activeThread?.environmentId}
+ projectId={activeThread?.projectId}
viewerRef={codeViewReveal.attach}
codeViewKey={codeViewMountKey}
className="h-full min-h-0 overflow-auto"
diff --git a/apps/web/src/components/chat/AssistantCitationChip.tsx b/apps/web/src/components/chat/AssistantCitationChip.tsx
index 2b90b09f68e..cf623580697 100644
--- a/apps/web/src/components/chat/AssistantCitationChip.tsx
+++ b/apps/web/src/components/chat/AssistantCitationChip.tsx
@@ -55,6 +55,7 @@ export function AssistantCitationChip({
const commentInputRef = useRef<HTMLTextAreaElement>(null);
const commentPopupRef = useRef<HTMLDivElement>(null);
const draftCommentRef = useRef<string | null>(null);
+ const [voiceSetupOpen, setVoiceSetupOpen] = useState(false);
const [unavailableSourceAnchor, setUnavailableSourceAnchor] =
useState<AssistantCitationSourceAnchor | null>(null);
const commentOpen = commentEditor?.open ?? false;
@@ -167,7 +168,7 @@ export function AssistantCitationChip({
{composerSourceLink}
{commentEditor ? (
<Popover
- open={commentEditor.open}
+ open={commentEditor.open && !voiceSetupOpen}
onOpenChange={(open, eventDetails) => {
if (!open && !settleDraftOnClose(eventDetails.reason)) {
eventDetails.cancel();
@@ -185,6 +186,8 @@ export function AssistantCitationChip({
</PopoverTrigger>
{commentEditor.open ? (
<PopoverPopup
+ // The editor stays mounted while its voice setup dialog replaces the popover.
+ keepMounted
{...composerFloatingLayerProps}
side={activeSourceAnchor ? "bottom" : "top"}
align="end"
@@ -217,6 +220,7 @@ export function AssistantCitationChip({
key={serializeAssistantCitation(citation)}
citation={citation}
inputRef={commentInputRef}
+ onVoiceSetupOpenChange={setVoiceSetupOpen}
onDraftChange={(comment) => {
draftCommentRef.current = comment;
commentEditor.onDraftChange?.(comment);
diff --git a/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx b/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx
index 0a8357bb9ba..cdd8e1249cf 100644
--- a/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx
+++ b/apps/web/src/components/chat/AssistantCitationCommentEditor.tsx
@@ -1,7 +1,10 @@
import { ASSISTANT_CITATION_MAX_COMMENT_LENGTH, type AssistantCitation } from "@t3tools/contracts";
-import { useState, type Ref } from "react";
+import { use, useEffect, useRef, useState, type RefObject } from "react";
+
+import { TextFieldSpeechControls, useTextFieldSpeech } from "~/speech/useTextFieldSpeech";
import { textDirection } from "~/textDirection";
+import { ComposerContextActionsContext } from "../composerContextActions";
import { Button } from "../ui/button";
export function AssistantCitationCommentEditor({
@@ -11,26 +14,55 @@ export function AssistantCitationCommentEditor({
onSubmitAndSend,
onCancel,
onDraftChange,
+ onVoiceSetupOpenChange,
}: {
citation: AssistantCitation;
- inputRef?: Ref<HTMLTextAreaElement>;
+ inputRef?: RefObject<HTMLTextAreaElement | null>;
onSubmit: (comment: string) => boolean;
onSubmitAndSend?: (comment: string) => boolean;
onCancel: () => void;
onDraftChange?: (comment: string) => void;
+ onVoiceSetupOpenChange?: (open: boolean) => void;
}) {
const [comment, setComment] = useState(citation.comment ?? "");
+ const localInputRef = useRef<HTMLTextAreaElement | null>(null);
+ const textareaRef = inputRef ?? localInputRef;
+ const { environmentId, projectId } = use(ComposerContextActionsContext);
+ const updateComment = (text: string) => {
+ setComment(text);
+ onDraftChange?.(text);
+ };
+ const speech = useTextFieldSpeech({
+ environmentId: environmentId ?? citation.environmentId,
+ projectId,
+ ownerKey: JSON.stringify([
+ "citation",
+ environmentId,
+ citation.threadId,
+ citation.messageId,
+ citation.start,
+ citation.end,
+ ]),
+ text: comment,
+ textareaRef,
+ onTextChange: updateComment,
+ });
+ const setupOpen = speech.setup.open;
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- keeps the citation popover open while the voice setup dialog owns focus
+ useEffect(() => {
+ onVoiceSetupOpenChange?.(setupOpen);
+ return () => onVoiceSetupOpenChange?.(false);
+ }, [onVoiceSetupOpenChange, setupOpen]);
const commentTooLong = comment.length > ASSISTANT_CITATION_MAX_COMMENT_LENGTH;
const submit = () => {
- if (!commentTooLong) onSubmit(comment);
+ speech.submitAfterDictation((text) => {
+ if (text.length <= ASSISTANT_CITATION_MAX_COMMENT_LENGTH) onSubmit(text);
+ });
};
const submitAndSend = () => {
- if (commentTooLong) return;
- if (onSubmitAndSend) {
- onSubmitAndSend(comment);
- } else {
- onSubmit(comment);
- }
+ speech.submitAfterDictation((text) => {
+ if (text.length <= ASSISTANT_CITATION_MAX_COMMENT_LENGTH) (onSubmitAndSend ?? onSubmit)(text);
+ });
};
return (
@@ -41,12 +73,14 @@ export function AssistantCitationCommentEditor({
if (event.nativeEvent.isComposing || event.keyCode === 229) return;
if (event.key === "Escape") {
event.preventDefault();
- onCancel();
+ if (speech.state.phase !== "idle") speech.cancel();
+ else onCancel();
}
}}
>
<textarea
- ref={inputRef}
+ ref={textareaRef}
+ readOnly={speech.freezesEditor}
aria-label="Comment on selected text"
aria-description="Enter to save the citation comment; Command/Ctrl+Enter to save and send; Shift+Enter for a new line."
aria-invalid={commentTooLong || undefined}
@@ -55,10 +89,7 @@ export function AssistantCitationCommentEditor({
dir={textDirection(comment)}
className="field-sizing-content block max-h-40 min-h-16 w-full resize-none bg-transparent px-1 py-1.5 text-base outline-none placeholder:text-muted-foreground sm:text-sm"
value={comment}
- onChange={(event) => {
- setComment(event.currentTarget.value);
- onDraftChange?.(event.currentTarget.value);
- }}
+ onChange={(event) => updateComment(event.currentTarget.value)}
onKeyDown={(event) => {
if (
event.key === "Enter" &&
@@ -81,7 +112,8 @@ export function AssistantCitationCommentEditor({
characters.
</p>
) : null}
- <div className="mt-2 flex items-center justify-end gap-2">
+ <div className="mt-2 flex flex-wrap items-center justify-end gap-2">
+ <TextFieldSpeechControls speech={speech} />
<Button
variant="outline"
size="xs"
@@ -92,7 +124,7 @@ export function AssistantCitationCommentEditor({
</Button>
<Button
size="xs"
- disabled={commentTooLong}
+ disabled={speech.preparing || (commentTooLong && !speech.blocksSubmission)}
onPointerDown={(event) => event.preventDefault()}
onClick={submit}
>
diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx
index e4515f759fd..69022a93fbe 100644
--- a/apps/web/src/components/chat/ChatComposer.tsx
+++ b/apps/web/src/components/chat/ChatComposer.tsx
@@ -1,3 +1,4 @@
+import { VoiceInputSetup } from "./VoiceInputSetup";
import { useComposerTypingGuard } from "./useComposerTypingGuard";
import { insertAttachmentReferencesForDraft } from "./composerAttachmentReferences";
import { DESKTOP_PASTE_AS_TEXT_EVENT } from "../../lib/desktopPasteAsText";
@@ -71,6 +72,7 @@ import {
type ComposerSubmissionIntent,
collapseExpandedComposerCursor,
composerRequiresModifier,
+ composerSubmissionIntentForKey,
composerStateAtPromptEnd,
detectComposerTrigger,
expandCollapsedComposerCursor,
@@ -135,6 +137,10 @@ import {
useAttachmentUploadStore,
} from "../../lib/attachmentUploadQueue";
import { attachmentUploadBlockReason } from "../../lib/attachmentUploadState";
+import { useDictationShortcut } from "../../speech/useDictationShortcut";
+import { useDictationSubmissionShortcut } from "../../speech/useDictationSubmissionShortcut";
+import { useEnvironmentSpeechInput } from "../../speech/useEnvironmentSpeechInput";
+import { ComposerSpeechButton, ComposerSpeechRecordingPill } from "./ComposerSpeechButton";
import type { AssistantCitationSourceAnchor } from "~/lib/assistantTextSelection";
import { shortcutLabelForCommand } from "../../keybindings";
import {
@@ -427,6 +433,7 @@ export interface ChatComposerProps {
activeThreadModelSelection: ModelSelection | null | undefined;
reportedModelSelection?: ModelSelection | null;
activeProjectId: ProjectId | null;
+ projectName?: string | undefined;
contextWindowEntries: Parameters<typeof deriveLatestContextWindowSnapshot>[0];
contextWindowLiveUsage: Parameters<typeof deriveLatestContextWindowSnapshot>[1];
@@ -554,6 +561,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
activeThreadModelSelection,
reportedModelSelection,
activeProjectId,
+ projectName,
contextWindowEntries,
contextWindowLiveUsage,
contextWindowProviderThread,
@@ -728,6 +736,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
const composerContextActions = {
environmentId,
cwd: gitCwd ?? undefined,
+ projectId: activeProjectId ?? undefined,
expandImage: (imageId: string) => {
const preview = buildExpandedImagePreview(composerImages, imageId);
if (preview) onExpandImage(preview);
@@ -1833,6 +1842,54 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
};
const readComposerSnapshotStable = useStableCallback(readComposerSnapshot);
+ const dictationDisabled =
+ !canOperateThread ||
+ environmentUnavailable !== null ||
+ projectSelectionRequired ||
+ isComposerApprovalState ||
+ activePendingIsResponding ||
+ activePendingProgress?.activeQuestion?.allowCustomAnswer === false;
+ const speechInput = useEnvironmentSpeechInput({
+ environmentId,
+ projectId: activeProjectId,
+ projectName,
+ ownerKey: questionAttachmentTarget ?? composerDraftTargetKey,
+ draftText: prompt,
+ disabled: dictationDisabled,
+ readDraft: () => {
+ const snapshot = readComposerSnapshot();
+ return {
+ text: snapshot.value,
+ selection: composerEditorRef.current?.readSelectionRange() ?? {
+ start: snapshot.expandedCursor,
+ end: snapshot.expandedCursor,
+ },
+ };
+ },
+ commitDraft: (text, selection) => {
+ const snapshot = readComposerSnapshot();
+ const applied = applyPromptReplacement(0, snapshot.value.length, text, {
+ expectedText: snapshot.value,
+ focusEditorAfterReplace: false,
+ });
+ if (!applied) return;
+ const cursor = collapseExpandedComposerCursor(text, selection.start, isLiteralPendingAnswer);
+ setComposerCursor(cursor);
+ window.requestAnimationFrame(() => composerEditorRef.current?.focusAt(cursor));
+ },
+ });
+ const { blocksSubmission: voiceInputActive, submitAfterDictation } = speechInput;
+ const voiceSendDisabledReason =
+ sendDisabledReason ?? (speechInput.preparing ? "Preparing voice input" : null);
+ const dictationShortcutLabel = useDictationShortcut({
+ keybindings,
+ speech: speechInput,
+ disabled: dictationDisabled,
+ terminalOpen,
+ modelPickerOpen: isComposerModelPickerOpen,
+ ...(backgroundThread ? { targetRef: composerFormRef } : {}),
+ });
+
const resolveActiveComposerTrigger = (): {
snapshot: { value: string; cursor: number; expandedCursor: number };
trigger: ComposerTrigger | null;
@@ -2076,11 +2133,19 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
return showPlanFollowUpPrompt || composerSendState.hasSendableContent;
};
+ const submitComposerRef = useRef<
+ (dispatchMode?: ComposerDispatchMode, submissionIntent?: ComposerSubmissionIntent) => void
+ >(() => {});
const submitComposer = (
event?: { preventDefault: () => void },
dispatchMode?: ComposerDispatchMode,
submissionIntent?: ComposerSubmissionIntent,
) => {
+ if (voiceInputActive) {
+ event?.preventDefault();
+ submitAfterDictation(() => submitComposerRef.current(dispatchMode, submissionIntent));
+ return;
+ }
if (
noProviderAvailable ||
isSendDisabled ||
@@ -2138,6 +2203,38 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
}
};
const submitComposerStable = useStableCallback(submitComposer);
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes the committed submission handler to the asynchronous dictation completion
+ useLayoutEffect(() => {
+ submitComposerRef.current = (mode, intent) => submitComposerStable(undefined, mode, intent);
+ }, [submitComposerStable]);
+ useDictationSubmissionShortcut({
+ enabled: voiceInputActive && !speechInput.preparing,
+ formRef: composerFormRef,
+ onKeyDown: (event) => {
+ const intent = composerSubmissionIntentForKey({
+ event,
+ keybindings,
+ isSoftwareKeyboardOpen,
+ isDraftThread: routeKind === "draft",
+ isRunning: phase === "running",
+ sendShortcut: settings.sendShortcut,
+ prompt: promptRef.current,
+ });
+ if (!intent) return;
+ event.preventDefault();
+ event.stopImmediatePropagation();
+ submitComposer(
+ undefined,
+ resolveComposerDispatchMode({
+ running: phase === "running",
+ alternateModifier: intent === "alternate",
+ canSteer: !isCompacting,
+ activeTurnDefault: followUpBehavior,
+ }),
+ intent,
+ );
+ },
+ });
const handleSubmitMessage = (event: React.MouseEvent<HTMLButtonElement>) => {
event.preventDefault();
submitComposer(
@@ -3444,7 +3541,8 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
disabled={
projectSelectionRequired ||
isChoiceOnlyPendingQuestion ||
- activePendingIsResponding
+ activePendingIsResponding ||
+ speechInput.freezesEditor
}
/>
</ComposerContextActionsContext>
@@ -3462,7 +3560,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
showPlanFollowUpPrompt={false}
promptHasText={false}
isSendBusy={isSendBusy}
- sendDisabledReason={sendDisabledReason}
+ sendDisabledReason={voiceSendDisabledReason}
isEnvironmentUnavailable={
environmentUnavailable !== null ||
noProviderAvailable ||
@@ -3484,13 +3582,25 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
message={providerInputSubmissionError ?? composerSubmissionError}
/>
- {isComposerCollapsedMobile ? null : (
+ {speechInput.preview ? (
+ <div
+ aria-label="Live transcription"
+ className="mx-3 mb-3 max-h-32 overflow-y-auto rounded-lg border border-border/60 bg-muted/30 px-3 py-2 text-sm leading-relaxed break-words whitespace-pre-wrap sm:mx-4"
+ >
+ <span>{speechInput.preview.committed}</span>
+ <span className="text-muted-foreground">{speechInput.preview.tentative}</span>
+ </div>
+ ) : null}
+
+ {isComposerCollapsedMobile && speechInput.state.phase === "idle" ? null : (
<div
data-chat-composer-footer="true"
className={cn(
"flex min-w-0 flex-nowrap items-center overflow-visible px-2 pb-2",
pendingUserInputs.length > 0 && "pt-2",
- showMobilePendingAnswerActions && "hidden sm:flex",
+ showMobilePendingAnswerActions &&
+ speechInput.state.phase === "idle" &&
+ "hidden sm:flex",
)}
>
{composerAttachButton ? <div className="me-2">{composerAttachButton}</div> : null}
@@ -3541,16 +3651,17 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
showPlanFollowUpPrompt={
pendingUserInputs.length === 0 && showPlanFollowUpPrompt
}
- promptHasText={prompt.trim().length > 0}
+ promptHasText={prompt.trim().length > 0 || voiceInputActive}
+ voiceInputActive={voiceInputActive}
isSendBusy={isSendBusy}
- sendDisabledReason={sendDisabledReason}
+ sendDisabledReason={voiceSendDisabledReason}
isEnvironmentUnavailable={
environmentUnavailable !== null ||
noProviderAvailable ||
projectSelectionRequired
}
isPreparingWorktree={isPreparingWorktree}
- hasSendableContent={composerSendState.hasSendableContent}
+ hasSendableContent={composerSendState.hasSendableContent || voiceInputActive}
canResume={showResumeAction}
preserveComposerFocusOnPointerDown={isMobileViewport}
isEditingQueuedMessage={isEditingQueuedMessage}
@@ -3567,7 +3678,45 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
{...(compactCommandAvailable
? { onCompactContext: compactThreadContextStable }
: {})}
- />
+ >
+ {speechInput.available ? (
+ <>
+ <ComposerSpeechRecordingPill
+ state={speechInput.state}
+ progress={null}
+ finishShortcutLabel={dictationShortcutLabel}
+ cancelShortcutLabel="Esc"
+ level={speechInput.level}
+ disabled={dictationDisabled}
+ onStart={() => void speechInput.start()}
+ onStop={() => void speechInput.stop()}
+ onCancel={() => speechInput.cancel()}
+ onSkipPostProcessing={() => speechInput.skipPostProcessing()}
+ />
+ <ComposerSpeechButton
+ state={speechInput.state}
+ progress={null}
+ shortcutLabel={dictationShortcutLabel}
+ disabled={dictationDisabled}
+ onStart={() => void speechInput.start()}
+ onCancel={() => speechInput.cancel()}
+ />
+ </>
+ ) : null}
+ <VoiceInputSetup
+ environmentId={speechInput.transcriptionEnvironmentId}
+ open={speechInput.setup.open}
+ step={speechInput.setup.step}
+ status={speechInput.status}
+ model={speechInput.setup.model}
+ downloading={speechInput.setup.downloading}
+ error={speechInput.setup.error}
+ onOpenChange={speechInput.setup.setOpen}
+ onDownload={() => void speechInput.setup.download()}
+ onCancelDownload={() => void speechInput.setup.cancelDownload()}
+ onStartRecording={() => void speechInput.setup.startRecording()}
+ />
+ </ComposerFooterPrimaryActions>
</div>
</div>
)}
diff --git a/apps/web/src/components/chat/ComposerFooterControls.tsx b/apps/web/src/components/chat/ComposerFooterControls.tsx
index 6ac13a51428..0a11e90a4cb 100644
--- a/apps/web/src/components/chat/ComposerFooterControls.tsx
+++ b/apps/web/src/components/chat/ComposerFooterControls.tsx
@@ -246,6 +246,7 @@ export function ComposerPermissionsSection(props: {
}
export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryActions(props: {
+ children?: React.ReactNode;
compact: boolean;
canOperateThread: boolean;
activeContextWindow: ContextWindowSnapshot | null;
@@ -266,6 +267,7 @@ export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryA
alternateShortcutLabel: string | null;
showPlanFollowUpPrompt: boolean;
promptHasText: boolean;
+ voiceInputActive?: boolean;
isSendBusy: boolean;
sendDisabledReason: string | null;
isEnvironmentUnavailable: boolean;
@@ -298,6 +300,7 @@ export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryA
) : props.reserveContextWindowMeter ? (
<ContextWindowMeterPlaceholder />
) : null}
+ {props.children}
<ComposerPrimaryActions
compact={props.compact}
canOperateThread={props.canOperateThread}
@@ -309,6 +312,7 @@ export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryA
alternateShortcutLabel={props.alternateShortcutLabel}
showPlanFollowUpPrompt={props.showPlanFollowUpPrompt}
promptHasText={props.promptHasText}
+ voiceInputActive={props.voiceInputActive ?? false}
isSendBusy={props.isSendBusy}
sendDisabledReason={props.sendDisabledReason}
isEnvironmentUnavailable={props.isEnvironmentUnavailable}
diff --git a/apps/web/src/components/chat/ComposerPrimaryActions.tsx b/apps/web/src/components/chat/ComposerPrimaryActions.tsx
index a4a8993a703..86d6cb69f9f 100644
--- a/apps/web/src/components/chat/ComposerPrimaryActions.tsx
+++ b/apps/web/src/components/chat/ComposerPrimaryActions.tsx
@@ -36,6 +36,8 @@ interface ComposerPrimaryActionsProps {
alternateShortcutLabel?: string | null;
showPlanFollowUpPrompt: boolean;
promptHasText: boolean;
+ /** Dictation is running; Send finishes it and submits once the transcript lands. */
+ voiceInputActive?: boolean;
isSendBusy: boolean;
sendDisabledReason: string | null;
isEnvironmentUnavailable: boolean;
@@ -90,6 +92,7 @@ export const ComposerPrimaryActions = memo(function ComposerPrimaryActions({
alternateShortcutLabel = null,
showPlanFollowUpPrompt,
promptHasText,
+ voiceInputActive = false,
isSendBusy,
sendDisabledReason,
isEnvironmentUnavailable,
@@ -186,8 +189,12 @@ export const ComposerPrimaryActions = memo(function ComposerPrimaryActions({
disabled={
!canOperateThread ||
isEnvironmentUnavailable ||
+ isSendDisabled ||
pendingAction.isResponding ||
- (pendingAction.isLastQuestion ? !pendingAction.isComplete : !pendingAction.canAdvance)
+ (!voiceInputActive &&
+ (pendingAction.isLastQuestion
+ ? !pendingAction.isComplete
+ : !pendingAction.canAdvance))
}
>
{formatPendingPrimaryActionLabel({
diff --git a/apps/web/src/components/chat/ComposerSpeechButton.tsx b/apps/web/src/components/chat/ComposerSpeechButton.tsx
new file mode 100644
index 00000000000..c0047b255ed
--- /dev/null
+++ b/apps/web/src/components/chat/ComposerSpeechButton.tsx
@@ -0,0 +1,302 @@
+import type { VoiceInputState } from "@t3tools/client-runtime/voice-input";
+import { CheckIcon, MicIcon, XIcon } from "lucide-react";
+import { useEffect, useState } from "react";
+
+import { VoiceInputErrorPill } from "./VoiceInputErrorPill";
+import { VoiceInputPill, VoiceWaveform } from "./VoiceInputPill";
+import { cn } from "~/lib/utils";
+import { Button } from "../ui/button";
+import { Spinner } from "../ui/spinner";
+import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
+
+type SpeechPresentation = {
+ status: string | null;
+ showsCancel: boolean;
+ showsConfirm: boolean;
+ confirmEnabled: boolean;
+ showsSend: boolean;
+};
+
+function resolveSpeechPresentation(
+ state: VoiceInputState,
+ progress: { downloaded: number; total: number } | null,
+): SpeechPresentation {
+ switch (state.phase) {
+ case "idle":
+ return {
+ status: null,
+ showsCancel: false,
+ showsConfirm: false,
+ confirmEnabled: false,
+ showsSend: true,
+ };
+ case "error":
+ return {
+ status: state.error ?? "Voice input failed",
+ showsCancel: false,
+ showsConfirm: false,
+ confirmEnabled: false,
+ showsSend: true,
+ };
+ case "preparing":
+ return {
+ status: progress
+ ? `Downloading speech model ${Math.round((progress.downloaded / Math.max(1, progress.total)) * 100)}%`
+ : "Preparing",
+ showsCancel: true,
+ showsConfirm: true,
+ confirmEnabled: false,
+ showsSend: true,
+ };
+ case "recording":
+ return {
+ status: "Recording",
+ showsCancel: true,
+ showsConfirm: true,
+ confirmEnabled: true,
+ showsSend: true,
+ };
+ case "transcribing":
+ return {
+ status: "Transcribing",
+ showsCancel: true,
+ showsConfirm: true,
+ confirmEnabled: false,
+ showsSend: true,
+ };
+ case "post-processing":
+ return {
+ status: "Post-processing",
+ showsCancel: true,
+ showsConfirm: true,
+ confirmEnabled: false,
+ showsSend: true,
+ };
+ }
+}
+
+function formatElapsed(seconds: number) {
+ return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
+}
+
+function RecordingStatus(props: { level: number; recordingLimitSeconds?: number | undefined }) {
+ const [elapsedSeconds, setElapsedSeconds] = useState(0);
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the timer displaying the native recording duration
+ useEffect(() => {
+ const startedAt = Date.now();
+ const timer = window.setInterval(
+ () => setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1_000)),
+ 250,
+ );
+ return () => window.clearInterval(timer);
+ }, []);
+
+ return (
+ <>
+ <VoiceWaveform level={props.level} />
+ <span aria-hidden className="size-1.5 shrink-0 rounded-full bg-primary" />
+ <span className="shrink-0 text-secondary-label text-xs tabular-nums">
+ {formatElapsed(elapsedSeconds)}
+ {props.recordingLimitSeconds !== undefined
+ ? ` / ${formatElapsed(props.recordingLimitSeconds)}`
+ : ""}
+ </span>
+ </>
+ );
+}
+
+function ComposerSpeechStatus(props: {
+ state: VoiceInputState;
+ progress: { downloaded: number; total: number } | null;
+ level: number;
+ recordingLimitSeconds?: number | undefined;
+}) {
+ const presentation = resolveSpeechPresentation(props.state, props.progress);
+
+ if (!presentation.status) return null;
+ const isRecording = props.state.phase === "recording";
+ const isError = props.state.phase === "error";
+
+ return (
+ <div
+ className="flex h-7 min-w-0 flex-1 items-center gap-2"
+ role="status"
+ aria-live={isRecording ? "off" : "polite"}
+ aria-label={presentation.status}
+ >
+ {isRecording ? (
+ <RecordingStatus level={props.level} recordingLimitSeconds={props.recordingLimitSeconds} />
+ ) : (
+ <span
+ className={cn(
+ "min-w-0 flex-1 truncate text-right text-sm text-secondary-label",
+ isError && "text-destructive",
+ )}
+ >
+ {presentation.status}
+ </span>
+ )}
+ </div>
+ );
+}
+
+function ComposerSpeechCancelButton(props: {
+ state: VoiceInputState;
+ shortcutLabel?: string | null;
+ onCancel(): void;
+}) {
+ if (props.state.phase === "idle") return null;
+ const label = props.state.phase === "error" ? "Dismiss voice input error" : "Cancel voice input";
+
+ return (
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <Button
+ type="button"
+ size="icon-sm-pill"
+ variant="ghost"
+ aria-label={label}
+ onPointerDown={(event) => event.preventDefault()}
+ onClick={props.onCancel}
+ className="shrink-0"
+ >
+ <XIcon />
+ </Button>
+ }
+ />
+ <TooltipPopup side="top">
+ {label}
+ {props.shortcutLabel ? ` (${props.shortcutLabel})` : ""}
+ </TooltipPopup>
+ </Tooltip>
+ );
+}
+
+export function ComposerSpeechButton(props: {
+ state: VoiceInputState;
+ progress: { downloaded: number; total: number } | null;
+ shortcutLabel?: string | null;
+ disabled?: boolean;
+ onStart(): void;
+ onCancel(): void;
+}) {
+ const presentation = resolveSpeechPresentation(props.state, props.progress);
+ const label = "Start voice input";
+
+ if (presentation.showsConfirm || props.state.phase === "error") return null;
+
+ return (
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <Button
+ type="button"
+ size="icon-sm"
+ variant="ghost"
+ aria-label={label}
+ disabled={props.disabled}
+ onPointerDown={(event) => event.preventDefault()}
+ onClick={() => {
+ if (props.disabled) return;
+ props.onStart();
+ }}
+ className="relative shrink-0"
+ >
+ <MicIcon />
+ </Button>
+ }
+ />
+ <TooltipPopup side="top">
+ {label}
+ {props.shortcutLabel ? ` (${props.shortcutLabel})` : ""}
+ </TooltipPopup>
+ </Tooltip>
+ );
+}
+
+export function ComposerSpeechRecordingPill(props: {
+ state: VoiceInputState;
+ progress: { downloaded: number; total: number } | null;
+ finishShortcutLabel?: string | null;
+ cancelShortcutLabel?: string | null;
+ level: number;
+ recordingLimitSeconds?: number | undefined;
+ onStart(): void;
+ disabled?: boolean;
+ onStop(): void;
+ onCancel(): void;
+ onSkipPostProcessing(): void;
+}) {
+ const presentation = resolveSpeechPresentation(props.state, props.progress);
+ if (props.state.phase === "error") {
+ return (
+ <VoiceInputErrorPill
+ error={props.state.error}
+ errorAction={props.state.errorAction}
+ disabled={props.disabled ?? false}
+ cancelShortcutLabel={props.cancelShortcutLabel ?? null}
+ onRetry={props.onStart}
+ onDismiss={props.onCancel}
+ />
+ );
+ }
+ if (!presentation.showsConfirm) return null;
+
+ const label = presentation.confirmEnabled
+ ? "Finish voice input"
+ : (presentation.status ?? "Voice input is busy");
+
+ return (
+ <VoiceInputPill>
+ <ComposerSpeechCancelButton
+ state={props.state}
+ shortcutLabel={props.cancelShortcutLabel ?? null}
+ onCancel={props.onCancel}
+ />
+ <ComposerSpeechStatus
+ state={props.state}
+ progress={props.progress}
+ level={props.level}
+ recordingLimitSeconds={props.recordingLimitSeconds}
+ />
+ {props.state.phase === "post-processing" ? (
+ <Button
+ type="button"
+ size="sm-pill"
+ variant="ghost"
+ onPointerDown={(event) => event.preventDefault()}
+ onClick={props.onSkipPostProcessing}
+ className="shrink-0"
+ >
+ Skip
+ </Button>
+ ) : (
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <Button
+ type="button"
+ size="icon-sm-pill"
+ aria-label={label}
+ disabled={!presentation.confirmEnabled}
+ onPointerDown={(event) => event.preventDefault()}
+ onClick={props.onStop}
+ className="shrink-0"
+ >
+ {presentation.confirmEnabled ? <CheckIcon /> : <Spinner aria-hidden />}
+ </Button>
+ }
+ />
+ <TooltipPopup side="top">
+ {label}
+ {presentation.confirmEnabled && props.finishShortcutLabel
+ ? ` (${props.finishShortcutLabel})`
+ : ""}
+ </TooltipPopup>
+ </Tooltip>
+ )}
+ </VoiceInputPill>
+ );
+}
diff --git a/apps/web/src/components/chat/RightPanelSurfaceContent.tsx b/apps/web/src/components/chat/RightPanelSurfaceContent.tsx
index b703e5d05da..f0d8afa5f31 100644
--- a/apps/web/src/components/chat/RightPanelSurfaceContent.tsx
+++ b/apps/web/src/components/chat/RightPanelSurfaceContent.tsx
@@ -50,6 +50,7 @@ interface RightPanelPullRequestProps {
}
interface RightPanelFileProps {
+ readonly projectId?: ProjectId | undefined;
readonly projectName: string | null;
readonly workspaceRoot: string | undefined;
readonly availableEditors: ReadonlyArray<EditorId>;
@@ -209,6 +210,7 @@ export function RightPanelSurfaceContent(props: {
environmentId={props.environmentId}
cwd={files.workspaceRoot ?? ""}
projectName={files.projectName ?? ""}
+ projectId={files.projectId}
threadRef={threadRef}
composerDraftTarget={props.composerDraftTarget}
keybindings={props.keybindings}
diff --git a/apps/web/src/components/chat/VoiceInputErrorPill.tsx b/apps/web/src/components/chat/VoiceInputErrorPill.tsx
new file mode 100644
index 00000000000..26f4895e1b3
--- /dev/null
+++ b/apps/web/src/components/chat/VoiceInputErrorPill.tsx
@@ -0,0 +1,115 @@
+import { MICROPHONE_PERMISSION_ERROR } from "@t3tools/client-runtime/voice-input";
+import { useNavigate } from "@tanstack/react-router";
+import { RotateCcwIcon, SettingsIcon, XIcon } from "lucide-react";
+
+import { Button } from "../ui/button";
+import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
+import { VoiceInputPill } from "./VoiceInputPill";
+
+const VOICE_ERROR_LABELS: Record<string, string> = {
+ [MICROPHONE_PERMISSION_ERROR]: "Microphone access needed",
+ "Microphone access is required for voice input.": "Microphone access needed",
+ "This draft is no longer available.": "Draft unavailable",
+ "Another voice recording is already active.": "Another recording active",
+ "Voice transcription is not available.": "Voice input unavailable",
+ "Voice transcription is still finishing. Try again shortly.": "Transcription still busy",
+ "Voice transcription is not available for this language.": "Language unsupported",
+ "Could not prepare voice transcription.": "Couldn’t prepare dictation",
+ "Could not start voice recording.": "Couldn’t start recording",
+ "Voice recording was interrupted.": "Recording interrupted",
+ "Could not finish voice recording.": "Couldn’t finish recording",
+ "Could not transcribe this recording.": "Transcription failed",
+ "Voice transcription on this environment failed.": "Transcription failed",
+ "The draft changed while voice input was running. The transcript was not added.": "Draft changed",
+ "No speech was detected.": "No speech detected",
+ "Microphone processing failed.": "Microphone processing failed",
+ "The microphone disconnected.": "Microphone disconnected",
+};
+
+function voiceErrorLabel(error: string | null): string {
+ return (error && VOICE_ERROR_LABELS[error]) || "Voice input failed";
+}
+
+export function VoiceInputErrorPill(props: {
+ error: string | null;
+ errorAction: "retry" | "settings" | null;
+ disabled?: boolean;
+ cancelShortcutLabel?: string | null;
+ onRetry(): void;
+ onDismiss(): void;
+}) {
+ const navigate = useNavigate();
+ const openSettings = props.errorAction === "settings";
+ const actionLabel = openSettings
+ ? "Open voice settings"
+ : props.error === "Could not transcribe this recording." ||
+ props.error === "Voice transcription on this environment failed."
+ ? "Retry transcription"
+ : "Record again";
+ return (
+ <VoiceInputPill>
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <Button
+ type="button"
+ size="icon-sm-pill"
+ variant="ghost"
+ aria-label="Dismiss voice input error"
+ onPointerDown={(event) => event.preventDefault()}
+ onClick={props.onDismiss}
+ >
+ <XIcon />
+ </Button>
+ }
+ />
+ <TooltipPopup side="top">
+ Dismiss voice input error
+ {props.cancelShortcutLabel ? ` (${props.cancelShortcutLabel})` : ""}
+ </TooltipPopup>
+ </Tooltip>
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <span
+ tabIndex={0}
+ role="status"
+ aria-label={props.error ?? "Voice input failed"}
+ className="min-w-0 flex-1 truncate text-sm text-destructive"
+ />
+ }
+ >
+ {voiceErrorLabel(props.error)}
+ </TooltipTrigger>
+ <TooltipPopup side="top">{props.error ?? "Voice input failed"}</TooltipPopup>
+ </Tooltip>
+ {props.errorAction ? (
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <Button
+ type="button"
+ size="icon-sm-pill"
+ variant="ghost"
+ aria-label={actionLabel}
+ disabled={props.disabled}
+ onPointerDown={(event) => event.preventDefault()}
+ onClick={() => {
+ if (openSettings) {
+ props.onDismiss();
+ void navigate({ to: "/settings/voice" });
+ } else {
+ props.onRetry();
+ }
+ }}
+ >
+ {openSettings ? <SettingsIcon /> : <RotateCcwIcon />}
+ </Button>
+ }
+ />
+ <TooltipPopup side="top">{actionLabel}</TooltipPopup>
+ </Tooltip>
+ ) : null}
+ </VoiceInputPill>
+ );
+}
diff --git a/apps/web/src/components/chat/VoiceInputPill.tsx b/apps/web/src/components/chat/VoiceInputPill.tsx
new file mode 100644
index 00000000000..a08567dbd67
--- /dev/null
+++ b/apps/web/src/components/chat/VoiceInputPill.tsx
@@ -0,0 +1,51 @@
+import { memo, useEffect, useRef, type ReactNode } from "react";
+import { useMediaQuery } from "~/hooks/useMediaQuery";
+
+const WAVEFORM_BAR_COUNT = 28;
+const WAVEFORM_BAR_IDS = Array.from({ length: WAVEFORM_BAR_COUNT }, (_, index) => String(index));
+
+export const VoiceWaveform = memo(function VoiceWaveform(props: { level: number }) {
+ const prefersReducedMotion = useMediaQuery("(prefers-reduced-motion: reduce)");
+ const barsRef = useRef<Array<HTMLSpanElement | null>>([]);
+ const levelsRef = useRef(Array<number>(WAVEFORM_BAR_COUNT).fill(0));
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- updates native waveform bar styles when microphone levels arrive
+ useEffect(() => {
+ const levels = levelsRef.current;
+ levels.copyWithin(0, 1);
+ levels[levels.length - 1] = props.level;
+ barsRef.current.forEach((bar, index) => {
+ if (!bar) return;
+ const level = levels[index] ?? 0;
+ bar.style.opacity = String(0.22 + level * 0.78);
+ bar.style.transform = `scaleY(${prefersReducedMotion ? 0.35 : Math.max(0.08, level)})`;
+ });
+ }, [prefersReducedMotion, props.level]);
+
+ return (
+ <div
+ aria-hidden
+ className="flex h-5 min-w-0 flex-1 items-center justify-between gap-px overflow-hidden"
+ >
+ {WAVEFORM_BAR_IDS.map((id, index) => (
+ <span
+ key={id}
+ ref={(bar) => {
+ barsRef.current[index] = bar;
+ }}
+ className="h-full w-0.5 shrink-0 origin-center rounded-full bg-primary opacity-25 transition-[transform,opacity] duration-100 ease-out motion-reduce:transition-none"
+ style={{ transform: `scaleY(${prefersReducedMotion ? 0.35 : 0.08})` }}
+ />
+ ))}
+ </div>
+ );
+});
+
+export function VoiceInputPill({ children }: { children: ReactNode }) {
+ return (
+ // Same height as the toolbar's icon buttons, so starting a recording never resizes the composer.
+ <div className="flex h-8 w-48 min-w-0 shrink-0 items-center gap-2 sm:h-7 sm:w-64">
+ {children}
+ </div>
+ );
+}
diff --git a/apps/web/src/components/chat/VoiceInputSetup.tsx b/apps/web/src/components/chat/VoiceInputSetup.tsx
new file mode 100644
index 00000000000..7b30783cf21
--- /dev/null
+++ b/apps/web/src/components/chat/VoiceInputSetup.tsx
@@ -0,0 +1,243 @@
+import { useStableCallback } from "~/hooks/useStableCallback";
+import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones";
+import type {
+ EnvironmentId,
+ EnvironmentSpeechModel,
+ EnvironmentSpeechStatus,
+} from "@t3tools/contracts";
+import { useNavigate } from "@tanstack/react-router";
+import { RefreshCwIcon } from "lucide-react";
+import { useEffect, useState } from "react";
+
+import { useClientSettings, updateClientSettings } from "../../hooks/useSettings";
+import { useEnvironment } from "../../state/environments";
+import { AuthSettingsWriteScope } from "@t3tools/contracts";
+import { useEnvironmentScope } from "../../state/session";
+import { Button } from "../ui/button";
+import { Dialog } from "../ui/dialog";
+import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
+import { WizardFooter, WizardHeader, WizardPanel, WizardPopup, WizardSteps } from "../ui/wizard";
+
+export function VoiceInputSetup(props: {
+ environmentId: EnvironmentId | null;
+ open: boolean;
+ step: number;
+ status: EnvironmentSpeechStatus | null;
+ model: EnvironmentSpeechModel | null;
+ downloading: boolean;
+ error: string | null;
+ onOpenChange(open: boolean): void;
+ onDownload(): void;
+ onCancelDownload(): void;
+ onStartRecording(): void;
+}) {
+ const navigate = useNavigate();
+ const environment = useEnvironment(props.environmentId);
+ const canManage = useEnvironmentScope(props.environmentId, AuthSettingsWriteScope);
+ const selectedMicrophone = useClientSettings((settings) => settings.voiceMicrophone);
+ const [microphones, setMicrophones] = useState<BrowserMicrophone[]>([]);
+ const [loadingMicrophones, setLoadingMicrophones] = useState(false);
+ const [microphoneError, setMicrophoneError] = useState<string | null>(null);
+ const refreshMicrophones = useStableCallback(async () => {
+ if (!navigator.mediaDevices?.enumerateDevices) return;
+ setLoadingMicrophones(true);
+ try {
+ setMicrophones(await listBrowserMicrophones());
+ setMicrophoneError(null);
+ } catch {
+ setMicrophoneError("Could not list microphones. You can still use the system default.");
+ }
+ setLoadingMicrophones(false);
+ });
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to native microphone changes while the device setup step is visible
+ useEffect(() => {
+ if (!props.open || props.step !== 1) return;
+ queueMicrotask(() => void refreshMicrophones());
+ const mediaDevices = navigator.mediaDevices;
+ mediaDevices?.addEventListener?.("devicechange", refreshMicrophones);
+ return () => mediaDevices?.removeEventListener?.("devicechange", refreshMicrophones);
+ }, [props.open, props.step, refreshMicrophones]);
+ const selectedIsUnavailable = Boolean(
+ selectedMicrophone && !microphones.some((device) => device.deviceId === selectedMicrophone),
+ );
+ const selectedMicrophoneLabel = selectedMicrophone
+ ? (microphones.find((device) => device.deviceId === selectedMicrophone)?.label ??
+ "Selected microphone (Unavailable)")
+ : "System default";
+ const status = props.status?.supported ? props.status : null;
+ const model = props.model;
+ const progress =
+ model?.downloaded === undefined ? null : Math.round((model.downloaded / model.size) * 100);
+ const openSettings = (hash: string) => {
+ props.onOpenChange(false);
+ void navigate({ to: "/settings/voice", hash });
+ };
+
+ return (
+ <Dialog open={props.open} onOpenChange={props.onOpenChange}>
+ <WizardPopup>
+ <WizardHeader
+ title="Set up voice input"
+ description={`Transcribe speech on ${environment?.label ?? "your transcription environment"} and use voice input across environments.`}
+ >
+ <WizardSteps steps={["Model", "Ready to dictate"]} currentStep={props.step} />
+ </WizardHeader>
+ <WizardPanel>
+ {props.step === 0 ? (
+ <section className="space-y-3 text-sm">
+ <div className="rounded-lg border border-border/70 bg-card/60 p-4">
+ <div className="flex items-center justify-between gap-2">
+ <h3 className="font-medium">{status?.model ?? "English transcription model"}</h3>
+ {status?.modelId === "handy-computer/parakeet-unified-en-0.6b-gguf" ? (
+ <span className="text-xs font-medium text-primary">
+ Recommended for English
+ </span>
+ ) : null}
+ </div>
+ <p className="mt-2 text-muted-foreground">
+ Download the {status ? Math.round(status.size / 1024 / 1024) : 697} MB model to{" "}
+ {environment?.label ?? "your transcription environment"}. Microphone audio is sent
+ there for transcription, as chosen in Voice settings.
+ </p>
+ <p className="mt-2 text-muted-foreground">
+ Silero (1.2 MB), used for silence filtering and pause detection, downloads
+ alongside the speech model. An installed copy is reused.
+ </p>
+ {props.downloading ? (
+ <div role="status" className="mt-4 space-y-1 text-muted-foreground">
+ <div
+ role="progressbar"
+ aria-label="Speech model download progress"
+ aria-valuemin={0}
+ aria-valuemax={100}
+ aria-valuenow={progress ?? undefined}
+ className="h-1 overflow-hidden rounded-full bg-muted"
+ >
+ <div
+ className="h-full bg-primary transition-[width] duration-200"
+ style={{ width: `${progress ?? 0}%` }}
+ />
+ </div>
+ <p className="text-xs">
+ {model?.state === "verifying"
+ ? "Verifying download…"
+ : progress === null
+ ? "Starting download…"
+ : `${progress}% downloaded`}
+ </p>
+ </div>
+ ) : null}
+ </div>
+ {props.error ? (
+ <p role="alert" className="text-destructive">
+ {props.error}
+ </p>
+ ) : null}
+ </section>
+ ) : (
+ <section className="space-y-4 text-sm">
+ <div className="space-y-2">
+ <label className="font-medium" htmlFor="voice-setup-microphone">
+ Microphone
+ </label>
+ <div className="flex items-center gap-2">
+ <Select
+ value={selectedMicrophone ? `device:${selectedMicrophone}` : "system-default"}
+ disabled={loadingMicrophones}
+ onValueChange={(value) => {
+ if (value)
+ void updateClientSettings({
+ voiceMicrophone:
+ value === "system-default" ? "" : value.slice("device:".length),
+ });
+ }}
+ >
+ <SelectTrigger
+ id="voice-setup-microphone"
+ size="sm"
+ aria-label="Microphone"
+ className="min-w-0 flex-1"
+ >
+ <SelectValue>{selectedMicrophoneLabel}</SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ <SelectItem value="system-default">System default</SelectItem>
+ {selectedIsUnavailable ? (
+ <SelectItem value={`device:${selectedMicrophone}`}>
+ Selected microphone (Unavailable)
+ </SelectItem>
+ ) : null}
+ {microphones.map((device) => (
+ <SelectItem key={device.deviceId} value={`device:${device.deviceId}`}>
+ {device.label}
+ </SelectItem>
+ ))}
+ </SelectPopup>
+ </Select>
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon-sm"
+ disabled={loadingMicrophones}
+ aria-label="Refresh microphones"
+ onClick={() => void refreshMicrophones()}
+ >
+ <RefreshCwIcon className="size-3.5" />
+ </Button>
+ </div>
+ {selectedIsUnavailable ? (
+ <p className="text-xs text-destructive">
+ The selected microphone is unavailable. Choose another or use the system
+ default.
+ </p>
+ ) : null}
+ {microphoneError ? (
+ <p role="alert" className="text-xs text-muted-foreground">
+ {microphoneError}
+ </p>
+ ) : null}
+ </div>
+ <p className="text-muted-foreground">
+ Speak and transcribe in multiple languages or translate into English, and get names
+ and technical terms right with the dictionary in Voice settings.
+ </p>
+ </section>
+ )}
+ </WizardPanel>
+ <WizardFooter>
+ {props.step === 0 ? (
+ <>
+ <Button variant="outline" onClick={() => openSettings("speech-models")}>
+ Choose another model or language
+ </Button>
+ {props.downloading ? (
+ <Button variant="outline" disabled={!canManage} onClick={props.onCancelDownload}>
+ Cancel download
+ </Button>
+ ) : null}
+ <Button
+ disabled={!canManage || props.downloading || !status}
+ onClick={props.onDownload}
+ >
+ {props.downloading
+ ? "Downloading…"
+ : props.error
+ ? "Retry download"
+ : canManage
+ ? "Download model"
+ : "Ask an environment admin to download"}
+ </Button>
+ </>
+ ) : (
+ <>
+ <Button variant="outline" onClick={() => openSettings("")}>
+ Customize in Voice settings
+ </Button>
+ <Button onClick={props.onStartRecording}>Start dictating</Button>
+ </>
+ )}
+ </WizardFooter>
+ </WizardPopup>
+ </Dialog>
+ );
+}
diff --git a/apps/web/src/components/composerContextActions.ts b/apps/web/src/components/composerContextActions.ts
new file mode 100644
index 00000000000..b5df31ea2b2
--- /dev/null
+++ b/apps/web/src/components/composerContextActions.ts
@@ -0,0 +1,22 @@
+import type { EnvironmentId, ProjectId } from "@t3tools/contracts";
+import { createContext, type MouseEvent } from "react";
+
+export interface ComposerContextActions {
+ environmentId: EnvironmentId | null;
+ cwd?: string | undefined;
+ projectId?: ProjectId | undefined;
+ expandImage: (imageId: string) => void;
+ expandVideo: (fileId: string) => void;
+ openFile: (fileId: string) => void;
+ openMention: (path: string) => void;
+ openPullRequest: (event: MouseEvent<HTMLElement>, url: string) => void;
+}
+
+export const ComposerContextActionsContext = createContext<ComposerContextActions>({
+ environmentId: null,
+ expandImage: () => {},
+ expandVideo: () => {},
+ openFile: () => {},
+ openMention: () => {},
+ openPullRequest: () => {},
+});
diff --git a/apps/web/src/components/composerContextPresentation.tsx b/apps/web/src/components/composerContextPresentation.tsx
index f8ac8222177..845e921725b 100644
--- a/apps/web/src/components/composerContextPresentation.tsx
+++ b/apps/web/src/components/composerContextPresentation.tsx
@@ -9,8 +9,8 @@ import type {
import { formatAttachmentSize } from "@t3tools/client-runtime/state/attachments";
import { videoMimeType } from "@t3tools/shared/video";
import { MessageCircleIcon, MousePointerClickIcon } from "lucide-react";
-import { createContext, type MouseEvent, type ReactElement, type ReactNode, use } from "react";
-import type { EnvironmentId } from "@t3tools/contracts";
+import { createContext, type ReactElement, type ReactNode, use } from "react";
+import { ComposerContextActionsContext } from "./composerContextActions";
import type { ComposerFileAttachment, ComposerImageAttachment } from "~/composerDraftStore";
import { composerFileNeedsReattach } from "~/composerDraftStore";
@@ -68,24 +68,10 @@ export type ComposerDraftContextRecord =
| { kind: "thread"; record: ThreadContextRecord }
| { kind: "environment"; record: EnvironmentContextRecord };
-export interface ComposerContextActions {
- environmentId: EnvironmentId | null;
- cwd?: string | undefined;
- expandImage: (imageId: string) => void;
- expandVideo: (fileId: string) => void;
- openFile: (fileId: string) => void;
- openMention: (path: string) => void;
- openPullRequest: (event: MouseEvent<HTMLElement>, url: string) => void;
-}
-
-export const ComposerContextActionsContext = createContext<ComposerContextActions>({
- environmentId: null,
- expandImage: () => {},
- expandVideo: () => {},
- openFile: () => {},
- openMention: () => {},
- openPullRequest: () => {},
-});
+export {
+ ComposerContextActionsContext,
+ type ComposerContextActions,
+} from "./composerContextActions";
export type ComposerDraftContextRecords = ReadonlyMap<string, ComposerDraftContextRecord>;
diff --git a/apps/web/src/components/diffs/AnnotatableCodeView.tsx b/apps/web/src/components/diffs/AnnotatableCodeView.tsx
index 81bcba75aa7..fdaea2df124 100644
--- a/apps/web/src/components/diffs/AnnotatableCodeView.tsx
+++ b/apps/web/src/components/diffs/AnnotatableCodeView.tsx
@@ -7,7 +7,7 @@ import type {
SelectedLineRange,
} from "@pierre/diffs";
import type { CodeViewHandle } from "@pierre/diffs/react";
-import type { ScopedThreadRef } from "@t3tools/contracts";
+import type { EnvironmentId, ProjectId, ScopedThreadRef } from "@t3tools/contracts";
import { useState, type ReactNode, type Ref } from "react";
import { type DraftId, useComposerDraftStore } from "~/composerDraftStore";
@@ -72,6 +72,8 @@ function appendAnnotationEntry(
}
interface AnnotatableCodeViewProps {
+ environmentId?: EnvironmentId | undefined;
+ projectId?: ProjectId | null | undefined;
codeViewKey: string;
files: ReadonlyArray<{
fileDiff: FileDiffMetadata;
@@ -104,6 +106,8 @@ interface DiffSelectionContext {
}
export function AnnotatableCodeView({
+ environmentId,
+ projectId,
codeViewKey,
files,
pendingFiles,
@@ -292,6 +296,14 @@ export function AnnotatableCodeView({
{annotation.metadata.entries.map((entry) => (
<DiffCommentAnnotation
key={entry.id}
+ environmentId={environmentId}
+ projectId={projectId}
+ ownerKey={JSON.stringify([
+ environmentId,
+ composerDraftTarget,
+ codeViewKey,
+ entry.id,
+ ])}
kind={entry.kind}
rangeLabel={entry.rangeLabel}
text={entry.kind === "draft" ? draftText : entry.text}
diff --git a/apps/web/src/components/diffs/DiffCommentAnnotation.tsx b/apps/web/src/components/diffs/DiffCommentAnnotation.tsx
index b202df0f52f..a73b631a888 100644
--- a/apps/web/src/components/diffs/DiffCommentAnnotation.tsx
+++ b/apps/web/src/components/diffs/DiffCommentAnnotation.tsx
@@ -1,8 +1,14 @@
+import type { EnvironmentId, ProjectId } from "@t3tools/contracts";
import { MessageCircle, Trash2 } from "lucide-react";
import { useLayoutEffect, useRef, useState, type ReactNode } from "react";
import { Button } from "~/components/ui/button";
import { Textarea } from "~/components/ui/textarea";
+import {
+ SpeechTextField,
+ TextFieldSpeechControls,
+ type TextFieldSpeech,
+} from "~/speech/useTextFieldSpeech";
import { textDirection } from "~/textDirection";
import { isCommentSubmitShortcut } from "./commentSubmitShortcut";
@@ -15,6 +21,10 @@ interface DiffCommentSecondaryAction {
}
interface DiffCommentAnnotationProps {
+ /** With an environment, the draft can be dictated; its project supplies vocabulary. */
+ environmentId?: EnvironmentId | undefined;
+ projectId?: ProjectId | null | undefined;
+ ownerKey?: string;
kind: "draft" | "comment";
rangeLabel: string;
text: string;
@@ -30,6 +40,9 @@ interface DiffCommentAnnotationProps {
}
export function DiffCommentAnnotation({
+ environmentId,
+ projectId,
+ ownerKey,
kind,
rangeLabel,
text,
@@ -88,8 +101,20 @@ export function DiffCommentAnnotation({
);
}
+ const submit = (
+ speech: TextFieldSpeech | undefined,
+ action: (text: string) => void,
+ allowEmpty = false,
+ ) => {
+ const finish = (value: string) => {
+ if (!pending && (allowEmpty || value.trim())) action(value.trim());
+ };
+ if (speech) speech.submitAfterDictation(finish);
+ else finish(displayedText);
+ };
+
// Pierre hides native carets and selection; textarea comments need their own visible feedback.
- return (
+ const renderDraft = (speech?: TextFieldSpeech) => (
<div
data-diff-comment-annotation
className="px-3 py-2 font-sans text-foreground caret-foreground selection:bg-primary selection:text-primary-foreground"
@@ -98,6 +123,7 @@ export function DiffCommentAnnotation({
>
<Textarea
ref={textareaRef}
+ readOnly={speech?.freezesEditor}
autoFocus={focusOnMount}
size="sm"
value={displayedText}
@@ -112,16 +138,18 @@ export function DiffCommentAnnotation({
onKeyDown={(event) => {
if (event.key === "Escape") {
event.preventDefault();
- onCancel();
+ if (speech && speech.state.phase !== "idle") speech.cancel();
+ else onCancel();
}
- if (isCommentSubmitShortcut(event, trimmedText, pending)) {
+ if (isCommentSubmitShortcut(event, trimmedText, pending, speech?.blocksSubmission)) {
event.preventDefault();
- onComment(trimmedText);
+ submit(speech, onComment);
}
}}
/>
- <div className="mt-1.5 flex items-center gap-1">
+ <div className="mt-1.5 flex flex-wrap items-center gap-1">
<span className="mr-auto text-3xs text-muted-foreground/70">⌘/Ctrl Enter to send</span>
+ {speech ? <TextFieldSpeechControls speech={speech} disabled={pending} /> : null}
<Button variant="ghost-muted" size="xs" onClick={onCancel}>
Cancel
</Button>
@@ -129,17 +157,41 @@ export function DiffCommentAnnotation({
<Button
size="xs"
variant="outline"
- disabled={!secondaryAction.allowEmpty && !trimmedText}
- onClick={() => secondaryAction.onAction(trimmedText)}
+ disabled={
+ speech?.preparing ||
+ pending ||
+ (!speech?.blocksSubmission && !secondaryAction.allowEmpty && !trimmedText)
+ }
+ onClick={() => submit(speech, secondaryAction.onAction, secondaryAction.allowEmpty)}
>
{secondaryAction.icon}
{secondaryAction.label}
</Button>
) : null}
- <Button size="xs" disabled={pending || !trimmedText} onClick={() => onComment(trimmedText)}>
+ <Button
+ size="xs"
+ disabled={speech?.preparing || pending || (!speech?.blocksSubmission && !trimmedText)}
+ onClick={() => submit(speech, onComment)}
+ >
{submitLabel}
</Button>
</div>
</div>
);
+
+ return environmentId ? (
+ <SpeechTextField
+ environmentId={environmentId}
+ projectId={projectId}
+ ownerKey={ownerKey ?? rangeLabel}
+ text={displayedText}
+ textareaRef={textareaRef}
+ onTextChange={onTextChange ?? setLocalDraftText}
+ disabled={pending}
+ >
+ {renderDraft}
+ </SpeechTextField>
+ ) : (
+ renderDraft()
+ );
}
diff --git a/apps/web/src/components/diffs/commentSubmitShortcut.ts b/apps/web/src/components/diffs/commentSubmitShortcut.ts
index a5034b62d44..8b6bb325035 100644
--- a/apps/web/src/components/diffs/commentSubmitShortcut.ts
+++ b/apps/web/src/components/diffs/commentSubmitShortcut.ts
@@ -8,8 +8,12 @@ export function isCommentSubmitShortcut(
event: CommentSubmitShortcutEvent,
value: string,
pending: boolean,
+ allowEmpty = false,
): boolean {
return (
- !pending && (event.metaKey || event.ctrlKey) && event.key === "Enter" && value.trim().length > 0
+ !pending &&
+ (event.metaKey || event.ctrlKey) &&
+ event.key === "Enter" &&
+ (allowEmpty || value.trim().length > 0)
);
}
diff --git a/apps/web/src/components/diffs/useReviewFilePatches.browser.test.tsx b/apps/web/src/components/diffs/useReviewFilePatches.browser.test.tsx
index d6a0622ea47..46edb4186a7 100644
--- a/apps/web/src/components/diffs/useReviewFilePatches.browser.test.tsx
+++ b/apps/web/src/components/diffs/useReviewFilePatches.browser.test.tsx
@@ -89,10 +89,10 @@ it("retains automatic and explicitly opened files through metadata insertion, re
diff: "diff --git a/f.txt b/f.txt\n--- a/f.txt\n+++ b/f.txt\n@@ -1 +1 @@\n-old\n+new\n",
files: tracked.files!.filter((f) => f.path === "f.txt"),
});
- act(() => registry.set(atom, loaded));
+ await act(() => registry.set(atom, loaded));
await expect.element(screen.getByText("f.txt:loaded")).toBeVisible();
const automatic = [...state.patches].find(([key]) => JSON.parse(key)[1] === "e.txt")![1];
- act(() =>
+ await act(() =>
registry.set(
automatic,
AsyncResult.success({
@@ -108,14 +108,14 @@ it("retains automatic and explicitly opened files through metadata insertion, re
);
await expect.element(screen.getByText("e.txt:loaded")).toBeVisible();
await expect.element(screen.getByText("f.txt:loaded")).toBeVisible();
- act(() =>
+ await act(() =>
registry.set(
atom,
AsyncResult.fail(new Error("refresh failed"), { previousSuccess: Option.some(loaded) }),
),
);
await expect.element(screen.getByText("f.txt:loaded")).toBeVisible();
- act(() => registry.set(atom, loaded));
+ await act(() => registry.set(atom, loaded));
await expect.element(screen.getByText("f.txt:loaded")).toBeVisible();
});
@@ -130,7 +130,7 @@ it("preserves deletion and addition patches for a file replaced by a symlink", a
</RegistryContext.Provider>,
);
const atom = [...state.patches.values()][0]!;
- act(() =>
+ await act(() =>
registry.set(
atom,
AsyncResult.success({
diff --git a/apps/web/src/components/files/FilePreviewPanel.tsx b/apps/web/src/components/files/FilePreviewPanel.tsx
index 9674bcfdebd..ed55da7c08b 100644
--- a/apps/web/src/components/files/FilePreviewPanel.tsx
+++ b/apps/web/src/components/files/FilePreviewPanel.tsx
@@ -5,6 +5,7 @@ import {
AuthPreviewOperateScope,
type EditorId,
type EnvironmentId,
+ type ProjectId,
type ResolvedKeybindingsConfig,
type ScopedThreadRef,
} from "@t3tools/contracts";
@@ -134,6 +135,7 @@ interface FilePreviewPanelProps {
environmentId: EnvironmentId;
cwd: string;
projectName: string;
+ projectId?: ProjectId | undefined;
relativePath: string | null;
attachment?: ChatFileAttachment;
threadRef: ScopedThreadRef;
@@ -650,6 +652,7 @@ function useEditableAfterHighlight(file: FileContents) {
interface EditableFileSurfaceProps {
environmentId: EnvironmentId;
+ projectId?: ProjectId | undefined;
cwd: string;
relativePath: string;
composerDraftTarget: ScopedThreadRef | DraftId;
@@ -668,6 +671,7 @@ interface FileSelectionOverride {
function EditableFileSurface({
environmentId,
+ projectId,
cwd,
relativePath,
composerDraftTarget,
@@ -897,6 +901,14 @@ function EditableFileSurface({
{annotation.metadata.entries.map((entry) => (
<DiffCommentAnnotation
key={entry.id}
+ environmentId={environmentId}
+ projectId={projectId}
+ ownerKey={JSON.stringify([
+ environmentId,
+ composerDraftTarget,
+ relativePath,
+ entry.id,
+ ])}
kind={entry.kind}
rangeLabel={formatFileCommentRange(entry.startLine, entry.endLine)}
text={entry.text}
@@ -989,6 +1001,7 @@ export default function FilePreviewPanel({
environmentId,
cwd,
projectName,
+ projectId,
relativePath: requestedPath,
attachment,
threadRef,
@@ -1511,6 +1524,7 @@ export default function FilePreviewPanel({
<EditableFileSurface
key={`${relativePath}:${resolvedTheme}`}
environmentId={environmentId}
+ projectId={projectId}
cwd={cwd}
relativePath={relativePath}
composerDraftTarget={composerDraftTarget}
diff --git a/apps/web/src/components/pullRequest/PullRequestCodeTab.browser.test.tsx b/apps/web/src/components/pullRequest/PullRequestCodeTab.browser.test.tsx
index ce6795cc8f4..cea33e74ffc 100644
--- a/apps/web/src/components/pullRequest/PullRequestCodeTab.browser.test.tsx
+++ b/apps/web/src/components/pullRequest/PullRequestCodeTab.browser.test.tsx
@@ -168,7 +168,6 @@ function pagedContainer() {
return container;
}
-// Rows are virtualized, so a header far down is not rendered until the view scrolls to it.
async function expectFileHeaderShown(
screen: Awaited<ReturnType<typeof render>>,
container: HTMLElement,
@@ -177,10 +176,10 @@ async function expectFileHeaderShown(
const header = screen.getByText(path, { exact: true }).first();
await expect.element(header).toBeInTheDocument();
await vi.waitFor(() => {
- const top = header.element().getBoundingClientRect().top;
- const bounds = container.getBoundingClientRect();
+ const bounds = container.querySelector(".diff-render-surface")!.getBoundingClientRect();
+ const { top, bottom } = header.element().getBoundingClientRect();
expect(top).toBeGreaterThanOrEqual(bounds.top);
- expect(top).toBeLessThan(bounds.top + bounds.height / 2);
+ expect(bottom).toBeLessThanOrEqual(bounds.bottom);
});
}
diff --git a/apps/web/src/components/pullRequest/PullRequestCodeTab.tsx b/apps/web/src/components/pullRequest/PullRequestCodeTab.tsx
index 8c551ee7c78..b8f46cd513b 100644
--- a/apps/web/src/components/pullRequest/PullRequestCodeTab.tsx
+++ b/apps/web/src/components/pullRequest/PullRequestCodeTab.tsx
@@ -1025,6 +1025,9 @@ function PullRequestCodeTab({
))}
{annotation.metadata.draft && draft && canSelectLines ? (
<DiffCommentAnnotation
+ environmentId={environmentId}
+ projectId={reference.projectId}
+ ownerKey={JSON.stringify([environmentId, reviewKey, draft])}
kind="draft"
rangeLabel={`${draft.path}:${getReviewPositionAnchor(draft.position).line}`}
text=""
diff --git a/apps/web/src/components/pullRequest/PullRequestCommentForm.tsx b/apps/web/src/components/pullRequest/PullRequestCommentForm.tsx
index 8a8ded649a3..0bb17925164 100644
--- a/apps/web/src/components/pullRequest/PullRequestCommentForm.tsx
+++ b/apps/web/src/components/pullRequest/PullRequestCommentForm.tsx
@@ -2,8 +2,9 @@ import { sourceControlClients } from "@t3tools/client-runtime/source-control-cli
import { useAtomCommand } from "~/state/use-atom-command";
import type { EnvironmentId, PullRequestDetailView, PullRequestRef } from "@t3tools/contracts";
import { SendIcon } from "lucide-react";
-import { useState, type RefObject } from "react";
+import { useEffect, useState, type RefObject } from "react";
+import { TextFieldSpeechControls, useTextFieldSpeech } from "~/speech/useTextFieldSpeech";
import { pullRequestEnvironment } from "~/state/pullRequests";
import { Button } from "../ui/button";
@@ -15,6 +16,7 @@ import { useInlineActionConfirm } from "./useInlineActionConfirm";
export function PullRequestCommentForm({
environmentId,
+ active,
reference,
detail,
actionPending,
@@ -24,6 +26,8 @@ export function PullRequestCommentForm({
onClose,
}: {
environmentId: EnvironmentId;
+ /** False while the form stays mounted but hidden; dictation is cancelled then. */
+ active: boolean;
reference: PullRequestRef;
detail: PullRequestDetailView;
actionPending: boolean;
@@ -37,6 +41,21 @@ export function PullRequestCommentForm({
}) {
const [body, setBody] = useState("");
const [submitting, setSubmitting] = useState<"comment" | "close" | "reopen" | null>(null);
+ const locked = submitting !== null || actionPending;
+ const speech = useTextFieldSpeech({
+ environmentId,
+ projectId: reference.projectId,
+ ownerKey: JSON.stringify(["pr-comment", environmentId, reference]),
+ text: body,
+ textareaRef,
+ onTextChange: setBody,
+ disabled: !active || locked,
+ });
+ const { cancel } = speech;
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- discards a recording when the hidden form can no longer be submitted
+ useEffect(() => {
+ if (!active) cancel();
+ }, [active, cancel]);
const postComment = useAtomCommand(pullRequestEnvironment.comment, {
reportFailure: false,
});
@@ -52,8 +71,8 @@ export function PullRequestCommentForm({
? ("reopen" as const)
: null;
- const submit = async (action: "comment" | "close" | "reopen") => {
- const trimmed = body.trim();
+ const submit = async (action: "comment" | "close" | "reopen", text: string) => {
+ const trimmed = text.trim();
if (trimmed.length === 0 || submitting !== null || actionPending) return;
setSubmitting(action);
if (action !== "comment") {
@@ -82,13 +101,18 @@ export function PullRequestCommentForm({
onClose();
onCommented();
};
+ const submitAfterDictation = (action: "comment" | "close" | "reopen") =>
+ speech.submitAfterDictation((text) => void submit(action, text));
+ const submitDisabled =
+ speech.preparing || (body.trim().length === 0 && !speech.blocksSubmission) || locked;
return (
<div className="space-y-2">
<Textarea
ref={textareaRef}
// Lock while posting because success clears the draft.
- disabled={submitting !== null || actionPending}
+ disabled={locked}
+ readOnly={speech.freezesEditor}
value={body}
rows={3}
placeholder="Leave a comment"
@@ -104,19 +128,22 @@ export function PullRequestCommentForm({
) {
event.preventDefault();
event.stopPropagation();
- if (!event.repeat) void submit("comment");
+ if (!event.repeat) submitAfterDictation("comment");
}
}}
/>
- <div className="flex flex-wrap justify-end gap-2">
+ <div className="flex flex-wrap items-center justify-end gap-2">
+ <div className="mr-auto flex min-w-0 flex-wrap items-center gap-1">
+ <TextFieldSpeechControls speech={speech} disabled={locked} />
+ </div>
{followUpAction === null ? null : (
<Button
size="xs"
variant={followUpAction === "close" ? "destructive-outline" : "outline"}
- disabled={body.trim().length === 0 || submitting !== null || actionPending}
+ disabled={submitDisabled}
{...(followUpAction === "close"
- ? inlineConfirm.props("close", () => void submit("close"))
- : { onClick: () => void submit("reopen") })}
+ ? inlineConfirm.props("close", () => submitAfterDictation("close"))
+ : { onClick: () => submitAfterDictation("reopen") })}
>
{followUpAction === "close" ? (
<PullRequestGlyph.closed className="size-3.5" />
@@ -147,8 +174,8 @@ export function PullRequestCommentForm({
<Button
size="xs"
variant="outline"
- disabled={body.trim().length === 0 || submitting !== null || actionPending}
- onClick={() => void submit("comment")}
+ disabled={submitDisabled}
+ onClick={() => submitAfterDictation("comment")}
>
<SendIcon className="size-3.5" />
{submitting === "comment" ? "Posting..." : "Comment"}
diff --git a/apps/web/src/components/pullRequest/PullRequestComposer.tsx b/apps/web/src/components/pullRequest/PullRequestComposer.tsx
index 5c747df739c..e787e02c022 100644
--- a/apps/web/src/components/pullRequest/PullRequestComposer.tsx
+++ b/apps/web/src/components/pullRequest/PullRequestComposer.tsx
@@ -151,6 +151,7 @@ export function PullRequestComposer({
{verdicts.length > 0 ? (
<div hidden={mode !== "review"}>
<PullRequestReviewForm
+ active={open && mode === "review"}
environmentId={environmentId}
reference={reference}
term={term}
@@ -171,6 +172,7 @@ export function PullRequestComposer({
{canComment ? (
<div hidden={mode !== "comment"}>
<PullRequestCommentForm
+ active={open && mode === "comment"}
environmentId={environmentId}
reference={reference}
detail={detail}
diff --git a/apps/web/src/components/pullRequest/PullRequestReviewAnnotation.tsx b/apps/web/src/components/pullRequest/PullRequestReviewAnnotation.tsx
index 57b50934b78..6bd09260875 100644
--- a/apps/web/src/components/pullRequest/PullRequestReviewAnnotation.tsx
+++ b/apps/web/src/components/pullRequest/PullRequestReviewAnnotation.tsx
@@ -14,6 +14,7 @@ import { settlePromise, squashAtomCommandFailure } from "@t3tools/client-runtime
import { RelativeTime } from "~/components/RelativeTime";
import { cn } from "~/lib/utils";
+import { SpeechTextField, TextFieldSpeechControls } from "~/speech/useTextFieldSpeech";
import { Button } from "../ui/button";
import { MorphIcon } from "~/components/MorphIcon";
@@ -36,6 +37,7 @@ const CARD_CLASS =
function submitKeys(input: {
readonly value: string;
readonly pending: boolean;
+ readonly allowEmpty?: boolean;
readonly onSubmit: () => void;
readonly onCancel?: (() => void) | undefined;
}) {
@@ -44,7 +46,7 @@ function submitKeys(input: {
event.preventDefault();
input.onCancel();
}
- if (isCommentSubmitShortcut(event, input.value, input.pending)) {
+ if (isCommentSubmitShortcut(event, input.value, input.pending, input.allowEmpty)) {
event.preventDefault();
input.onSubmit();
}
@@ -146,6 +148,7 @@ export function ReviewThreadCard({
const [editingId, setEditingId] = useState<string | null>(null);
const [savingEdit, setSavingEdit] = useState(false);
const sendingRef = useRef(false);
+ const replyRef = useRef<HTMLTextAreaElement>(null);
const [loadedPage, setLoadedPage] = useState<
(PullRequestThreadCommentsResult & { readonly threadId: string }) | null
>(null);
@@ -174,8 +177,8 @@ export function ReviewThreadCard({
}
};
- const send = async () => {
- const trimmed = reply.trim();
+ const send = async (text: string) => {
+ const trimmed = text.trim();
if (trimmed.length === 0 || pending || sendingRef.current) return;
sendingRef.current = true;
const result = await settlePromise(() => onReply(trimmed));
@@ -323,36 +326,55 @@ export function ReviewThreadCard({
{canReply ? (
replying ? (
- <div className="mt-2">
- <Textarea
- autoFocus
- size="sm"
- value={reply}
- placeholder="Reply"
- aria-label="Reply to this conversation"
- onChange={(event) => setReply(event.target.value)}
- onKeyDown={(event) =>
- submitKeys({
- value: reply,
- pending,
- onSubmit: () => void send(),
- onCancel: () => setReplying(false),
- })(event)
- }
- />
- <div className="mt-2 flex justify-end gap-2">
- <Button size="xs" variant="ghost" onClick={() => setReplying(false)}>
- Cancel
- </Button>
- <Button
- size="xs"
- disabled={pending || reply.trim().length === 0}
- onClick={() => void send()}
- >
- Reply
- </Button>
- </div>
- </div>
+ <SpeechTextField
+ environmentId={environmentId}
+ projectId={reference.projectId}
+ ownerKey={JSON.stringify(["pr-reply", environmentId, reference, thread.id])}
+ text={reply}
+ textareaRef={replyRef}
+ onTextChange={setReply}
+ disabled={pending}
+ >
+ {(speech) => (
+ <div className="mt-2">
+ <Textarea
+ ref={replyRef}
+ readOnly={speech.freezesEditor}
+ autoFocus
+ size="sm"
+ value={reply}
+ placeholder="Reply"
+ aria-label="Reply to this conversation"
+ onChange={(event) => setReply(event.target.value)}
+ onKeyDown={submitKeys({
+ value: reply,
+ pending,
+ allowEmpty: speech.blocksSubmission,
+ onSubmit: () => speech.submitAfterDictation((text) => void send(text)),
+ onCancel: () =>
+ speech.state.phase === "idle" ? setReplying(false) : speech.cancel(),
+ })}
+ />
+ <div className="mt-2 flex flex-wrap items-center justify-end gap-2">
+ <TextFieldSpeechControls speech={speech} disabled={pending} />
+ <Button size="xs" variant="ghost" onClick={() => setReplying(false)}>
+ Cancel
+ </Button>
+ <Button
+ size="xs"
+ disabled={
+ speech.preparing ||
+ pending ||
+ (!speech.blocksSubmission && reply.trim().length === 0)
+ }
+ onClick={() => speech.submitAfterDictation((text) => void send(text))}
+ >
+ Reply
+ </Button>
+ </div>
+ </div>
+ )}
+ </SpeechTextField>
) : (
<Button size="xs" variant="ghost" className="mt-2" onClick={() => setReplying(true)}>
Reply
diff --git a/apps/web/src/components/pullRequest/PullRequestReviewForm.browser.test.tsx b/apps/web/src/components/pullRequest/PullRequestReviewForm.browser.test.tsx
new file mode 100644
index 00000000000..d8901b6db6c
--- /dev/null
+++ b/apps/web/src/components/pullRequest/PullRequestReviewForm.browser.test.tsx
@@ -0,0 +1,54 @@
+import { createRef } from "react";
+import { EnvironmentId, ProjectId } from "@t3tools/contracts";
+import { expect, it, vi } from "vite-plus/test";
+import { act, render } from "../../../test/render";
+import { PullRequestReviewForm } from "./PullRequestReviewForm";
+import { pullRequestReviewKey, usePullRequestReviewStore } from "./pullRequestReviewStore";
+
+const mocks = vi.hoisted(() => ({
+ submit: vi.fn().mockResolvedValue({ _tag: "Success" }),
+ queue: vi.fn<(submit: (text: string) => void) => void>(),
+}));
+vi.mock("~/speech/useTextFieldSpeech", () => ({
+ TextFieldSpeechControls: () => null,
+ useTextFieldSpeech: () => ({ blocksSubmission: true, submitAfterDictation: mocks.queue }),
+}));
+vi.mock("~/state/use-atom-command", () => ({ useAtomCommand: () => mocks.submit }));
+vi.mock("~/state/pullRequests", () => ({ pullRequestEnvironment: { submitReview: {} } }));
+
+it("submits only the line comments still in the draft when dictation finishes", async () => {
+ mocks.submit.mockClear();
+ mocks.queue.mockClear();
+ const reference = { projectId: ProjectId.make("test"), repository: "test/repo", number: 1 };
+ const key = pullRequestReviewKey(reference);
+ const store = usePullRequestReviewStore.getState();
+ store.clear(key);
+ store.addComment(key, {
+ id: "removed",
+ body: "Do not post",
+ path: "a.ts",
+ position: { kind: "added", newLine: 1 },
+ });
+ await using screen = await render(
+ <PullRequestReviewForm
+ active
+ environmentId={EnvironmentId.make("test")}
+ reference={reference}
+ term="pull request"
+ verdicts={["comment"]}
+ requestChangesSummaryRequired={false}
+ textareaRef={createRef()}
+ pending={false}
+ onPendingChange={vi.fn()}
+ onSubmitted={vi.fn()}
+ />,
+ );
+ await screen.getByRole("button", { name: "Submit review" }).click();
+ await act(() => store.clear(key));
+ await act(() => mocks.queue.mock.calls[0]![0]("Dictated summary"));
+ expect(mocks.submit).toHaveBeenCalledWith(
+ expect.objectContaining({
+ input: expect.objectContaining({ body: "Dictated summary", comments: [] }),
+ }),
+ );
+});
diff --git a/apps/web/src/components/pullRequest/PullRequestReviewForm.tsx b/apps/web/src/components/pullRequest/PullRequestReviewForm.tsx
index 57c933256ff..b70b59f9d69 100644
--- a/apps/web/src/components/pullRequest/PullRequestReviewForm.tsx
+++ b/apps/web/src/components/pullRequest/PullRequestReviewForm.tsx
@@ -2,11 +2,13 @@ import { useAtomCommand } from "~/state/use-atom-command";
import { capitalizeTerm } from "@t3tools/client-runtime/source-control-clients";
import type { EnvironmentId, PullRequestRef, PullRequestReviewVerdict } from "@t3tools/contracts";
import { CheckIcon, MessageSquareIcon, XCircleIcon } from "lucide-react";
-import { useState, type ReactNode, type RefObject } from "react";
+import { useEffect, useState, type ReactNode, type RefObject } from "react";
+import { TextFieldSpeechControls, useTextFieldSpeech } from "~/speech/useTextFieldSpeech";
import { pullRequestEnvironment } from "~/state/pullRequests";
import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime";
+import { isCommentSubmitShortcut } from "../diffs/commentSubmitShortcut";
import { Button } from "../ui/button";
import { Select, SelectItem, SelectPopup, SelectTrigger } from "../ui/select";
import { Textarea } from "../ui/textarea";
@@ -46,6 +48,7 @@ const VERDICTS: ReadonlyArray<{
export function PullRequestReviewForm({
environmentId,
+ active,
reference,
term,
verdicts,
@@ -56,6 +59,8 @@ export function PullRequestReviewForm({
onSubmitted,
}: {
environmentId: EnvironmentId;
+ /** False while the form stays mounted but hidden; dictation is cancelled then. */
+ active: boolean;
reference: PullRequestRef;
/** The host's word for a change request, as in "pull request". */
term: string;
@@ -72,6 +77,20 @@ export function PullRequestReviewForm({
const body = usePullRequestReviewStore((store) => store.summaries[reviewKey] ?? "");
const removeComments = usePullRequestReviewStore((store) => store.removeComments);
const setSummary = usePullRequestReviewStore((store) => store.setSummary);
+ const speech = useTextFieldSpeech({
+ environmentId,
+ projectId: reference.projectId,
+ ownerKey: JSON.stringify(["pr-review", environmentId, reviewKey]),
+ text: body,
+ textareaRef,
+ onTextChange: (text) => setSummary(reviewKey, text),
+ disabled: !active || pending,
+ });
+ const { cancel } = speech;
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- discards a recording when the hidden form can no longer be submitted
+ useEffect(() => {
+ if (!active) cancel();
+ }, [active, cancel]);
const clearSummary = usePullRequestReviewStore((store) => store.clearSummary);
const submitReview = useAtomCommand(pullRequestEnvironment.submitReview, {
reportFailure: false,
@@ -81,10 +100,10 @@ export function PullRequestReviewForm({
const selectedVerdict =
offered.find((verdict) => verdict.value === requestedVerdict) ?? offered[0];
- const submit = async (verdict: (typeof VERDICTS)[number]) => {
- if (pending) return;
- const submittedBody = body;
- const submittedComments = comments;
+ const submit = async (verdict: (typeof VERDICTS)[number], text: string) => {
+ const submittedComments = usePullRequestReviewStore.getState().drafts[reviewKey] ?? [];
+ if (pending || !canSubmit(verdict.value, text, submittedComments.length)) return;
+ const submittedBody = text;
onPendingChange(true);
const result = await submitReview({
environmentId,
@@ -118,12 +137,12 @@ export function PullRequestReviewForm({
};
// Forgejo requires a summary when requesting changes, even with inline comments.
- const canSubmit = (verdict: PullRequestReviewVerdict, summary: string) =>
+ const canSubmit = (verdict: PullRequestReviewVerdict, summary: string, commentCount: number) =>
verdict === "request-changes" && requestChangesSummaryRequired
? summary.trim().length > 0
- : verdict === "approve" || summary.trim().length > 0 || comments.length > 0;
+ : verdict === "approve" || summary.trim().length > 0 || commentCount > 0;
const placeholder =
- selectedVerdict === undefined || canSubmit(selectedVerdict.value, "")
+ selectedVerdict === undefined || canSubmit(selectedVerdict.value, "", comments.length)
? "Summarize your review (optional)"
: selectedVerdict.value === "request-changes" && requestChangesSummaryRequired
? "Summarize your review (required to request changes)"
@@ -134,12 +153,27 @@ export function PullRequestReviewForm({
<Textarea
ref={textareaRef}
rows={3}
+ readOnly={speech.freezesEditor}
value={body}
placeholder={placeholder}
aria-label="Review summary"
onChange={(event) => setSummary(reviewKey, event.target.value)}
+ onKeyDown={(event) => {
+ if (event.nativeEvent.isComposing || event.keyCode === 229 || event.repeat) return;
+ if (
+ selectedVerdict !== undefined &&
+ isCommentSubmitShortcut(event, body, pending || speech.preparing, true)
+ ) {
+ event.preventDefault();
+ event.stopPropagation();
+ speech.submitAfterDictation((text) => void submit(selectedVerdict, text));
+ }
+ }}
/>
- <div className="mt-2 flex justify-end gap-2">
+ <div className="mt-2 flex flex-wrap items-center justify-end gap-2">
+ <div className="mr-auto flex min-w-0 flex-wrap items-center gap-1">
+ <TextFieldSpeechControls speech={speech} disabled={pending} />
+ </div>
{offered.length > 1 ? (
<Select
value={selectedVerdict?.value ?? null}
@@ -148,7 +182,7 @@ export function PullRequestReviewForm({
if (value !== null) setRequestedVerdict(value);
}}
>
- <SelectTrigger size="xs" className="mr-auto w-auto min-w-0" aria-label="Review verdict">
+ <SelectTrigger size="xs" className="w-auto min-w-0" aria-label="Review verdict">
<span className="flex items-center gap-1.5">
{selectedVerdict?.icon}
{selectedVerdict?.label}
@@ -169,10 +203,14 @@ export function PullRequestReviewForm({
<Button
size="xs"
disabled={
- pending || selectedVerdict === undefined || !canSubmit(selectedVerdict.value, body)
+ speech.preparing ||
+ pending ||
+ selectedVerdict === undefined ||
+ (!canSubmit(selectedVerdict.value, body, comments.length) && !speech.blocksSubmission)
}
onClick={() => {
- if (selectedVerdict !== undefined) void submit(selectedVerdict);
+ if (selectedVerdict !== undefined)
+ speech.submitAfterDictation((text) => void submit(selectedVerdict, text));
}}
>
{pending ? "Submitting..." : "Submit review"}
diff --git a/apps/web/src/components/settings/KeybindingsSettings.logic.ts b/apps/web/src/components/settings/KeybindingsSettings.logic.ts
index fe9b47b4dca..c40286d86fb 100644
--- a/apps/web/src/components/settings/KeybindingsSettings.logic.ts
+++ b/apps/web/src/components/settings/KeybindingsSettings.logic.ts
@@ -373,6 +373,7 @@ export function commandLabel(command: KeybindingCommand): string {
if (command === "composer.sendAlternate") return "Composer: Opposite Queue or Steer Action";
if (command === "composer.sendBackground") return "Composer: Start in Background";
if (command === "composer.sendAndNewThread") return "Composer: Send and Start New Thread";
+ if (command === "composer.dictation") return "Composer: Start or Finish Dictation";
if (command === "thread.steerQueuedMessage") return "Queue: Send First Queued Message";
if (command === "thread.editQueuedMessage") return "Queue: Edit Last Queued Message";
if (command === "thread.copyReference") return "Pull Request: Copy Link or Thread";
diff --git a/apps/web/src/components/settings/MicrophoneTest.tsx b/apps/web/src/components/settings/MicrophoneTest.tsx
new file mode 100644
index 00000000000..d3597e2dea6
--- /dev/null
+++ b/apps/web/src/components/settings/MicrophoneTest.tsx
@@ -0,0 +1,350 @@
+import { MICROPHONE_PERMISSION_ERROR } from "@t3tools/client-runtime/voice-input";
+import { CheckIcon, MicIcon, PauseIcon, PlayIcon, RotateCcwIcon, XIcon } from "lucide-react";
+import { useEffect, useRef, useState } from "react";
+
+import { useStableCallback } from "~/hooks/useStableCallback";
+import workletUrl from "../../speech/pcmWorklet.ts?worker&url";
+import { VoiceInputErrorPill } from "../chat/VoiceInputErrorPill";
+import { VoiceInputPill, VoiceWaveform } from "../chat/VoiceInputPill";
+import { Button } from "../ui/button";
+
+const LOOPBACK_SECONDS = 10;
+const formatTime = (seconds: number) =>
+ `${Math.floor(seconds / 60)}:${String(Math.floor(seconds % 60)).padStart(2, "0")}`;
+
+type Capture = {
+ stream?: MediaStream;
+ recorder?: MediaRecorder;
+ context?: AudioContext;
+ worklet?: AudioWorkletNode;
+ timer?: ReturnType<typeof setTimeout>;
+ elapsedTimer?: ReturnType<typeof setInterval>;
+ startedAt?: number;
+};
+
+/**
+ * Records a few seconds and plays them back in the browser. The audio never leaves this
+ * device: no environment, transcriber, or provider is involved, so a silent result points
+ * at the microphone rather than at transcription.
+ */
+export function MicrophoneTest({ microphoneId }: { microphoneId: string }) {
+ const [phase, setPhase] = useState<"idle" | "starting" | "recording" | "ready">("idle");
+ const [recording, setRecording] = useState<{ url: string; duration: number } | null>(null);
+ const [error, setError] = useState<string | null>(null);
+ const [level, setLevel] = useState(0);
+ const [elapsed, setElapsed] = useState(0);
+ const capture = useRef<Capture | null>(null);
+ const recordingUrl = useRef<string | null>(null);
+
+ const releaseCapture = useStableCallback(() => {
+ const current = capture.current;
+ if (!current) return;
+ capture.current = null;
+ clearTimeout(current.timer);
+ clearInterval(current.elapsedTimer);
+ current.worklet?.disconnect();
+ current.worklet?.port.close();
+ if (current.recorder && current.recorder.state !== "inactive") current.recorder.stop();
+ current.stream?.getTracks().forEach((track) => track.stop());
+ void current.context?.close().catch(() => undefined);
+ });
+ const discard = () => {
+ if (recordingUrl.current) URL.revokeObjectURL(recordingUrl.current);
+ recordingUrl.current = null;
+ setRecording(null);
+ setPhase("idle");
+ };
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- releases the microphone and the recording's object URL on unmount
+ useEffect(
+ () => () => {
+ releaseCapture();
+ if (recordingUrl.current) URL.revokeObjectURL(recordingUrl.current);
+ },
+ [releaseCapture],
+ );
+
+ const start = async () => {
+ releaseCapture();
+ discard();
+ setError(null);
+ setLevel(0);
+ setElapsed(0);
+ setPhase("starting");
+ if (
+ !navigator.mediaDevices?.getUserMedia ||
+ typeof MediaRecorder === "undefined" ||
+ typeof AudioWorkletNode === "undefined"
+ ) {
+ setError("Microphone testing is unavailable in this browser. Use HTTPS or the desktop app.");
+ setPhase("idle");
+ return;
+ }
+ const current: Capture = {};
+ capture.current = current;
+ try {
+ const stream = await navigator.mediaDevices.getUserMedia({
+ audio: microphoneId ? { deviceId: { exact: microphoneId } } : true,
+ });
+ if (capture.current !== current) {
+ stream.getTracks().forEach((track) => track.stop());
+ return;
+ }
+ current.stream = stream;
+ const recorder = new MediaRecorder(stream);
+ current.recorder = recorder;
+ const chunks: Blob[] = [];
+ recorder.addEventListener("dataavailable", ({ data }) => {
+ if (capture.current === current && data.size) chunks.push(data);
+ });
+ recorder.addEventListener("stop", () => {
+ if (capture.current !== current) return;
+ const blob = new Blob(chunks, { type: recorder.mimeType });
+ const duration =
+ current.startedAt === undefined ? 0 : (performance.now() - current.startedAt) / 1_000;
+ releaseCapture();
+ setLevel(0);
+ if (!blob.size) {
+ setError("No audio was recorded. Try again.");
+ setPhase("idle");
+ return;
+ }
+ recordingUrl.current = URL.createObjectURL(blob);
+ setRecording({ url: recordingUrl.current, duration });
+ setPhase("ready");
+ });
+ const onRecordingError = () => {
+ if (capture.current !== current) return;
+ releaseCapture();
+ setLevel(0);
+ setError("Could not record microphone audio. Try again.");
+ setPhase("idle");
+ };
+ recorder.addEventListener("error", onRecordingError);
+ for (const track of stream.getAudioTracks()) {
+ track.addEventListener(
+ "ended",
+ () => {
+ if (capture.current !== current) return;
+ setError("The microphone disconnected.");
+ if (recorder.state !== "inactive") recorder.stop();
+ },
+ { once: true },
+ );
+ }
+ const context = new AudioContext();
+ current.context = context;
+ await context.audioWorklet.addModule(workletUrl);
+ if (capture.current !== current) return;
+ const worklet = new AudioWorkletNode(context, "t3-pcm-capture", {
+ channelCount: 1,
+ processorOptions: { meterOnly: true },
+ });
+ current.worklet = worklet;
+ worklet.port.onmessage = ({ data }: MessageEvent<number | "stopped">) => {
+ if (capture.current === current && typeof data === "number") setLevel(data);
+ };
+ worklet.onprocessorerror = onRecordingError;
+ context.createMediaStreamSource(stream).connect(worklet);
+ worklet.connect(context.destination);
+ await context.resume();
+ if (capture.current !== current) return;
+ const startedAt = performance.now();
+ current.startedAt = startedAt;
+ current.elapsedTimer = setInterval(() => {
+ setElapsed(Math.min(LOOPBACK_SECONDS, Math.floor((performance.now() - startedAt) / 1_000)));
+ }, 100);
+ recorder.start();
+ worklet.port.postMessage("start", []);
+ setPhase("recording");
+ current.timer = setTimeout(() => {
+ if (recorder.state !== "inactive") recorder.stop();
+ }, LOOPBACK_SECONDS * 1_000);
+ } catch (cause) {
+ if (capture.current !== current) return;
+ releaseCapture();
+ setPhase("idle");
+ setError(
+ typeof cause === "object" &&
+ cause !== null &&
+ "name" in cause &&
+ cause.name === "NotAllowedError"
+ ? MICROPHONE_PERMISSION_ERROR
+ : cause instanceof Error
+ ? cause.message
+ : "Could not access the microphone.",
+ );
+ }
+ };
+ const cancel = () => {
+ releaseCapture();
+ discard();
+ setError(null);
+ };
+
+ if (error) {
+ return (
+ <VoiceInputErrorPill
+ error={error}
+ errorAction="retry"
+ onRetry={() => void start()}
+ onDismiss={cancel}
+ />
+ );
+ }
+ if (phase === "idle") {
+ return (
+ <Button size="sm" variant="outline" onClick={() => void start()}>
+ <MicIcon />
+ Test microphone
+ </Button>
+ );
+ }
+ if (phase === "ready" && recording) {
+ return (
+ <MicrophonePlayback
+ key={recording.url}
+ src={recording.url}
+ duration={recording.duration}
+ onRetry={() => void start()}
+ onDone={cancel}
+ />
+ );
+ }
+ return (
+ <VoiceInputPill>
+ <Button
+ size="icon-sm-pill"
+ variant="ghost"
+ aria-label="Cancel microphone test"
+ onClick={cancel}
+ >
+ <XIcon />
+ </Button>
+ {phase === "recording" ? (
+ <>
+ <VoiceWaveform level={level} />
+ <span aria-hidden className="size-1.5 shrink-0 rounded-full bg-primary" />
+ <span
+ className="shrink-0 text-xs tabular-nums text-muted-foreground"
+ aria-label="Recording time"
+ >
+ {formatTime(elapsed)} / {formatTime(LOOPBACK_SECONDS)}
+ </span>
+ <meter
+ aria-label="Microphone input level"
+ min={0}
+ max={1}
+ value={level}
+ className="sr-only"
+ />
+ </>
+ ) : (
+ <span role="status" className="min-w-0 flex-1 truncate text-xs text-muted-foreground">
+ Opening microphone…
+ </span>
+ )}
+ <Button
+ size="icon-sm-pill"
+ aria-label="Stop recording"
+ disabled={phase !== "recording"}
+ onClick={() => {
+ const recorder = capture.current?.recorder;
+ if (recorder && recorder.state !== "inactive") recorder.stop();
+ }}
+ >
+ <CheckIcon />
+ </Button>
+ </VoiceInputPill>
+ );
+}
+
+function MicrophonePlayback({
+ src,
+ duration,
+ onRetry,
+ onDone,
+}: {
+ src: string;
+ duration: number;
+ onRetry(): void;
+ onDone(): void;
+}) {
+ const audio = useRef<HTMLAudioElement>(null);
+ const [playing, setPlaying] = useState(false);
+ const [position, setPosition] = useState(0);
+ const [failed, setFailed] = useState(false);
+ const progress = duration > 0 ? Math.min(position / duration, 1) * 100 : 0;
+
+ const togglePlayback = async () => {
+ const player = audio.current;
+ if (!player) return;
+ if (!player.paused) {
+ player.pause();
+ return;
+ }
+ setFailed(false);
+ if (player.ended) player.currentTime = 0;
+ try {
+ await player.play();
+ } catch {
+ setFailed(true);
+ }
+ };
+
+ return (
+ <div className="flex flex-col gap-1">
+ <audio
+ ref={audio}
+ src={src}
+ preload="metadata"
+ onPlay={() => setPlaying(true)}
+ onPause={() => setPlaying(false)}
+ onEnded={() => setPlaying(false)}
+ onTimeUpdate={() => setPosition(audio.current?.currentTime ?? 0)}
+ onError={() => setFailed(true)}
+ />
+ <VoiceInputPill>
+ <Button
+ size="icon-sm-pill"
+ variant="ghost"
+ aria-label={playing ? "Pause recording" : "Play recording"}
+ onClick={() => void togglePlayback()}
+ >
+ {playing ? <PauseIcon /> : <PlayIcon />}
+ </Button>
+ <input
+ type="range"
+ aria-label="Recording playback position"
+ min={0}
+ max={duration}
+ step={0.01}
+ value={Math.min(position, duration)}
+ style={{
+ background: `linear-gradient(to right, var(--primary) ${progress}%, var(--muted) ${progress}%)`,
+ }}
+ disabled={duration <= 0}
+ onChange={(event) => {
+ const next = Number(event.target.value);
+ if (audio.current) audio.current.currentTime = next;
+ setPosition(next);
+ }}
+ className="h-1 min-w-0 flex-1 cursor-pointer appearance-none rounded-full bg-muted accent-primary focus-visible:outline-2 focus-visible:-outline-offset-2 focus-visible:outline-ring [&::-webkit-slider-thumb]:size-2.5 [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-primary [&::-moz-range-thumb]:size-2.5 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:border-0 [&::-moz-range-thumb]:bg-primary"
+ />
+ <span className="shrink-0 text-xs tabular-nums text-muted-foreground">
+ {formatTime(position)}
+ </span>
+ <Button size="icon-sm-pill" variant="ghost" aria-label="Record again" onClick={onRetry}>
+ <RotateCcwIcon />
+ </Button>
+ <Button size="icon-sm-pill" aria-label="Done" onClick={onDone}>
+ <CheckIcon />
+ </Button>
+ </VoiceInputPill>
+ {failed ? (
+ <p role="alert" className="text-xs text-destructive">
+ Could not play this recording. Try recording again.
+ </p>
+ ) : null}
+ </div>
+ );
+}
diff --git a/apps/web/src/components/settings/SettingsSidebarNav.tsx b/apps/web/src/components/settings/SettingsSidebarNav.tsx
index a9c8b01f077..e273d0c672e 100644
--- a/apps/web/src/components/settings/SettingsSidebarNav.tsx
+++ b/apps/web/src/components/settings/SettingsSidebarNav.tsx
@@ -23,6 +23,7 @@ import {
PanelsTopLeftIcon,
KeyboardIcon,
KeyRoundIcon,
+ MicIcon,
PaletteIcon,
SearchIcon,
ServerIcon,
@@ -88,6 +89,7 @@ const SETTINGS_SECTION_ICONS: Readonly<
> = {
"/settings/general": Settings2Icon,
"/settings/diagnostics": WrenchIcon,
+ "/settings/voice": MicIcon,
"/settings/appearance": PaletteIcon,
"/settings/chat": MessageSquareTextIcon,
"/settings/diffs": FileDiffIcon,
diff --git a/apps/web/src/components/settings/SpeechLanguageSelect.tsx b/apps/web/src/components/settings/SpeechLanguageSelect.tsx
new file mode 100644
index 00000000000..948412184b4
--- /dev/null
+++ b/apps/web/src/components/settings/SpeechLanguageSelect.tsx
@@ -0,0 +1,61 @@
+import type { EnvironmentSpeechStatus } from "@t3tools/contracts";
+import { effectiveSpeechLanguage } from "@t3tools/shared/speech";
+import { updateClientSettings, useClientSettings } from "../../hooks/useSettings";
+import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
+
+const languageNames = new Intl.DisplayNames(["en"], { type: "language" });
+export const languageLabel = (code: string) => languageNames.of(code) ?? code;
+
+type ModelLanguages = Pick<
+ Extract<EnvironmentSpeechStatus, { supported: true }>,
+ "languages" | "supportsLanguageDetection"
+>;
+
+export function SpeechLanguageSelect({ model }: { readonly model: ModelLanguages | null }) {
+ const preference = useClientSettings((settings) => settings.speechLanguage);
+ const languages = model?.languages ?? [];
+ const resolved = model
+ ? effectiveSpeechLanguage(
+ { languages, supportsLanguageDetection: model.supportsLanguageDetection ?? false },
+ preference,
+ )
+ : preference;
+ const effectiveLabel = resolved === "auto" ? "Auto-detect" : languageLabel(resolved);
+ const unavailable =
+ preference !== "auto" && preference !== resolved
+ ? `${languageLabel(preference)} unavailable`
+ : null;
+ const label = unavailable
+ ? `${effectiveLabel} (${unavailable})`
+ : preference === "auto" && resolved !== "auto"
+ ? `Auto (${effectiveLabel})`
+ : effectiveLabel;
+ return (
+ <Select
+ value={preference}
+ disabled={!model}
+ onValueChange={(value) => {
+ if (value) void updateClientSettings({ speechLanguage: value });
+ }}
+ >
+ <SelectTrigger size="sm" aria-label="Transcription language" className="max-w-64">
+ <SelectValue>{label}</SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ <SelectItem value="auto">Auto</SelectItem>
+ {unavailable ? (
+ <SelectItem value={preference} disabled>
+ {unavailable}
+ </SelectItem>
+ ) : null}
+ {languages
+ .toSorted((left, right) => languageLabel(left).localeCompare(languageLabel(right)))
+ .map((code) => (
+ <SelectItem key={code} value={code}>
+ {languageLabel(code)}
+ </SelectItem>
+ ))}
+ </SelectPopup>
+ </Select>
+ );
+}
diff --git a/apps/web/src/components/settings/SpeechModelComparison.tsx b/apps/web/src/components/settings/SpeechModelComparison.tsx
new file mode 100644
index 00000000000..425073a006f
--- /dev/null
+++ b/apps/web/src/components/settings/SpeechModelComparison.tsx
@@ -0,0 +1,126 @@
+import type { EnvironmentSpeechModel } from "@t3tools/contracts";
+import { CheckIcon } from "lucide-react";
+
+import { cn } from "../../lib/utils";
+import { languageLabel } from "./SpeechLanguageSelect";
+
+const commonLanguages = [
+ "en",
+ "zh",
+ "hi",
+ "es",
+ "fr",
+ "bn",
+ "pt",
+ "id",
+ "ur",
+ "ru",
+ "de",
+ "ja",
+ "mr",
+ "vi",
+ "te",
+ "sw",
+ "ha",
+ "tr",
+ "tl",
+ "ta",
+ "jw",
+ "ko",
+ "am",
+ "th",
+ "it",
+ "gu",
+ "kn",
+ "yo",
+];
+
+export function orderSpeechLanguages(languages: readonly string[], preferred: readonly string[]) {
+ const order = [...new Set([...preferred, ...commonLanguages])];
+ const rank = new Map(order.map((code, index) => [code, index]));
+ return languages.toSorted(
+ (left, right) =>
+ (rank.get(left) ?? order.length) - (rank.get(right) ?? order.length) ||
+ languageLabel(left).localeCompare(languageLabel(right)),
+ );
+}
+
+export function LanguageRow({
+ label,
+ selected,
+ onClick,
+}: {
+ label: string;
+ selected: boolean;
+ onClick: () => void;
+}) {
+ return (
+ <button
+ type="button"
+ aria-pressed={selected}
+ onClick={onClick}
+ className={cn(
+ "flex shrink-0 cursor-pointer items-center justify-between gap-2 rounded-md px-2.5 py-1.5 text-left text-sm outline-none hover:bg-muted/50 focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-ring",
+ selected && "bg-muted font-medium",
+ )}
+ >
+ {label}
+ {selected && <CheckIcon className="size-3.5 shrink-0" />}
+ </button>
+ );
+}
+
+export function ModelBenchmarks({
+ model,
+ languages,
+ facts,
+}: {
+ model: EnvironmentSpeechModel;
+ languages: readonly string[];
+ /** Plain facts, such as language count and size, shown after the scores. */
+ facts: readonly string[];
+}) {
+ const data = model.benchmark;
+ const visible = !data
+ ? []
+ : languages.length
+ ? languages
+ : [model.languages.find((code) => data.accuracy[code]) ?? model.languages[0]].filter(
+ (code): code is string => code !== undefined,
+ );
+ return (
+ <div className="mt-1 flex flex-wrap gap-x-3 gap-y-1 text-xs text-muted-foreground tabular-nums">
+ {visible.map((code) => {
+ const score = data?.accuracy[code];
+ return (
+ <span key={code}>
+ {languageLabel(code)}:{" "}
+ {score ? (
+ <>
+ <strong className="font-medium text-foreground">{score.grade}</strong> (
+ {score.errorPercent}% error)
+ </>
+ ) : (
+ "not benchmarked"
+ )}
+ </span>
+ );
+ })}
+ {data ? (
+ <span>
+ Speed:{" "}
+ {data.secondsPer30s === null ? (
+ "not benchmarked"
+ ) : (
+ <>
+ <strong className="font-medium text-foreground">{data.secondsPer30s} s</strong> / 30 s
+ </>
+ )}
+ </span>
+ ) : null}
+ {facts.map((fact) => (
+ <span key={fact}>{fact}</span>
+ ))}
+ </div>
+ );
+}
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
new file mode 100644
index 00000000000..ab1c0187c28
--- /dev/null
+++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
@@ -0,0 +1,319 @@
+import { PrimaryConnectionTarget } from "@t3tools/client-runtime/connection";
+import {
+ EnvironmentId,
+ type EnvironmentSpeechModel,
+ type EnvironmentSpeechStatus,
+} from "@t3tools/contracts";
+import { expect, it, vi } from "vite-plus/test";
+import { userEvent } from "vite-plus/test/browser";
+import { render } from "../../../test/render";
+import { __setClientSettingsForTests, getClientSettings } from "../../hooks/useSettings";
+import {
+ useEnvironmentSpeechInput,
+ useVoiceTranscriptionDestination,
+} from "../../speech/useEnvironmentSpeechInput";
+import { chooseAudioDestination, chooseTranscriptCleanup } from "../../speech/voiceConsent";
+import { toastManager } from "../ui/toast";
+import { SettingsScopeProvider } from "./SettingsScopeContext";
+import { SpeechLanguageSelect } from "./SpeechLanguageSelect";
+import { VoiceSettingsPanel } from "./VoiceSettingsPanel";
+
+vi.mock(import("../../speech/useEnvironmentSpeechInput"), { spy: true });
+
+it("preserves the model test transcript when its model is filtered out", async () => {
+ const target = new PrimaryConnectionTarget({
+ environmentId: EnvironmentId.make("voice-filter"),
+ label: "Test environment",
+ httpBaseUrl: window.location.origin,
+ wsBaseUrl: "ws://localhost",
+ });
+ const prepared = { ...target, target, socketUrl: target.wsBaseUrl, httpAuthorization: null };
+ const model: EnvironmentSpeechModel = {
+ id: "tiny",
+ name: "Tiny",
+ description: "",
+ size: 100,
+ languages: ["en"],
+ accuracy: 1,
+ speed: 1,
+ recommended: false,
+ supportsStreaming: false,
+ supportsTranslation: false,
+ supportsLanguageDetection: false,
+ active: true,
+ state: "installed",
+ };
+ const status = {
+ supported: true,
+ state: "ready",
+ modelId: model.id,
+ model: model.name,
+ size: model.size,
+ supportsStreaming: false,
+ supportsTranslation: false,
+ acceleration: "cpu",
+ modelUnloadTimeout: "never",
+ gpuDevices: [],
+ } satisfies EnvironmentSpeechStatus;
+ const fetch = globalThis.fetch;
+ using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
+ const url = new URL(input instanceof Request ? input.url : input, window.location.href);
+ if (url.pathname === "/api/voice/status") return Promise.resolve(Response.json(status));
+ if (url.pathname === "/api/voice/models")
+ return Promise.resolve(
+ Response.json({
+ models: [
+ model,
+ {
+ ...model,
+ id: "both",
+ name: "Bilingual",
+ active: false,
+ languages: ["en", "fr"],
+ ...(url.searchParams.has("benchmarks")
+ ? {
+ benchmark: {
+ accuracy: {
+ en: { grade: "A-", errorPercent: 4 },
+ fr: { grade: "B+", errorPercent: 5.5 },
+ },
+ secondsPer30s: 2.1,
+ },
+ }
+ : {}),
+ },
+ ],
+ }),
+ );
+ return fetch(input, init);
+ });
+ const transcript = "Keep this transcript.";
+ await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
+ () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }),
+ async () => {
+ await vi.mocked(useEnvironmentSpeechInput).withImplementation(
+ (input) => ({
+ available: true,
+ status,
+ transcriptionEnvironmentId: target.environmentId,
+ transcriptionEnvironmentLabel: target.label,
+ setup: {
+ open: false,
+ step: 0,
+ model: null,
+ downloading: false,
+ error: null,
+ setOpen: () => {},
+ download: async () => {},
+ cancelDownload: async () => {},
+ startRecording: async () => {},
+ },
+ state: { phase: "idle", error: null, errorAction: null },
+ preview: null,
+ level: 0,
+ preparing: false,
+ blocksSubmission: false,
+ freezesEditor: false,
+ start: async () => {
+ input.commitDraft(transcript, { start: transcript.length, end: transcript.length });
+ },
+ submitAfterDictation: () => {},
+ stop: async () => {},
+ cancel: () => {},
+ skipPostProcessing: () => {},
+ }),
+ async () => {
+ await using screen = await render(
+ <SettingsScopeProvider search={{}} onChange={() => {}}>
+ <VoiceSettingsPanel />
+ </SettingsScopeProvider>,
+ );
+ const testModel = screen.getByRole("button", { name: "Test model", exact: true });
+ await expect.element(testModel).toBeVisible();
+ await userEvent.click(testModel);
+ const result = screen.getByRole("status", { name: "Test transcript" });
+ await expect.element(result).toHaveTextContent(transcript);
+
+ const search = screen.getByRole("textbox", { name: "Search transcription models" });
+ await userEvent.fill(search, "no matching model");
+ await expect.element(screen.getByText("No models match.")).toBeVisible();
+ await userEvent.clear(search);
+ await userEvent.fill(screen.getByRole("textbox", { name: "Search languages" }), "fr");
+ const french = screen.getByRole("button", { name: "French", exact: true });
+ await userEvent.click(french);
+ await expect.element(screen.getByText("Bilingual", { exact: true })).toBeVisible();
+ await expect.element(screen.getByText("B+", { exact: true })).toBeVisible();
+ expect(screen.container.querySelector("#transcription-test")).toHaveAttribute("hidden");
+ await userEvent.click(french);
+ await expect.element(result).toBeVisible();
+ await vi.waitFor(() =>
+ expect(
+ screen.container.querySelector('[aria-label="Test transcript"]')?.textContent,
+ ).toBe(transcript),
+ );
+ },
+ );
+ },
+ );
+});
+
+it("recovers from a failed download progress poll without notifying, but reports a failed initial load", async () => {
+ const target = new PrimaryConnectionTarget({
+ environmentId: EnvironmentId.make("voice-progress"),
+ label: "Test environment",
+ httpBaseUrl: window.location.origin,
+ wsBaseUrl: "ws://localhost",
+ });
+ const prepared = { ...target, target, socketUrl: target.wsBaseUrl, httpAuthorization: null };
+ const model: EnvironmentSpeechModel = {
+ id: "tiny",
+ name: "Tiny",
+ description: "",
+ size: 100,
+ languages: ["en"],
+ accuracy: 1,
+ speed: 1,
+ recommended: false,
+ supportsStreaming: false,
+ supportsTranslation: false,
+ supportsLanguageDetection: false,
+ active: false,
+ state: "downloading",
+ };
+ const status = {
+ supported: true,
+ state: "missing-model",
+ modelId: model.id,
+ model: model.name,
+ size: model.size,
+ supportsStreaming: false,
+ supportsTranslation: false,
+ acceleration: "cpu",
+ modelUnloadTimeout: "never",
+ gpuDevices: [],
+ };
+ const unavailable = () => new Response(null, { status: 503 });
+ const readModels = vi
+ .fn((_withBenchmarks: boolean) => unavailable())
+ .mockImplementationOnce(() => Response.json({ models: [model] }))
+ .mockImplementationOnce(unavailable)
+ .mockImplementationOnce(() => Response.json({ models: [{ ...model, state: "installed" }] }));
+ const fetch = globalThis.fetch;
+ using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
+ const url = new URL(input instanceof Request ? input.url : input, window.location.href);
+ if (url.pathname === "/api/voice/status") return Promise.resolve(Response.json(status));
+ if (url.pathname === "/api/voice/models")
+ return Promise.resolve(readModels(url.searchParams.has("benchmarks")));
+ return fetch(input, init);
+ });
+ using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");
+ await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
+ () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }),
+ async () => {
+ const panel = (key: string) => (
+ <SettingsScopeProvider search={{}} onChange={() => {}}>
+ <VoiceSettingsPanel key={key} />
+ </SettingsScopeProvider>
+ );
+ await using screen = await render(panel("first"));
+ await expect.element(screen.getByRole("button", { name: "Use model" })).toBeVisible();
+ expect(readModels.mock.calls.map(([withBenchmarks]) => withBenchmarks)).toEqual([
+ true,
+ false,
+ false,
+ ]);
+ expect(notify).not.toHaveBeenCalled();
+ await screen.rerender(panel("reopened"));
+ await vi.waitFor(() =>
+ expect(notify).toHaveBeenCalledWith(
+ expect.objectContaining({ title: "Could not load transcription models" }),
+ ),
+ );
+ },
+ );
+});
+
+it.each([
+ ["speech-models", "voice-transcription-destination"],
+ ["speech-hardware", "voice-transcription-destination"],
+ ["transcription-test", "voice-transcription-destination"],
+ ["speech-correction-word", "speech-post-processing"],
+ ["speech-post-processing-model", "speech-post-processing"],
+])(
+ "searching for disabled %s focuses its opt-in without enabling it",
+ async (target, prerequisite) => {
+ chooseAudioDestination(null);
+ chooseTranscriptCleanup(false);
+ await using screen = await render(
+ <SettingsScopeProvider search={{}} onChange={() => {}}>
+ <VoiceSettingsPanel />
+ </SettingsScopeProvider>,
+ { router: `/#${target}` },
+ );
+ await vi.waitFor(() => expect(document.activeElement?.id).toBe(prerequisite));
+ await expect
+ .element(screen.getByRole("combobox", { name: "Send microphone audio to" }))
+ .toHaveTextContent("Off");
+ await expect
+ .element(screen.getByRole("switch", { name: "Clean up transcripts with a provider" }))
+ .not.toBeChecked();
+ },
+);
+
+it("keeps a missing destination selected without claiming it cannot transcribe", async () => {
+ __setClientSettingsForTests(getClientSettings());
+ using cleanup = new DisposableStack();
+ cleanup.defer(() => chooseAudioDestination(null));
+ chooseAudioDestination({ kind: "environment", environmentId: EnvironmentId.make("missing") });
+ await using screen = await render(
+ <SettingsScopeProvider search={{}} onChange={() => {}}>
+ <VoiceSettingsPanel />
+ </SettingsScopeProvider>,
+ );
+ await expect
+ .element(screen.getByRole("combobox", { name: "Send microphone audio to" }))
+ .toHaveTextContent("Chosen environment (unavailable)");
+ await expect.element(screen.getByText(/can’t transcribe audio/)).not.toBeInTheDocument();
+});
+
+it("shows the model's effective language and restores the saved preference when supported again", async () => {
+ const previous = getClientSettings();
+ using cleanup = new DisposableStack();
+ cleanup.defer(() => __setClientSettingsForTests(previous));
+ __setClientSettingsForTests({ ...previous, speechLanguage: "fr" });
+ await using screen = await render(<SpeechLanguageSelect model={{ languages: ["en"] }} />);
+ const picker = screen.getByRole("combobox", { name: "Transcription language" });
+ await expect.element(picker).toHaveTextContent("English (French unavailable)");
+ await screen.rerender(
+ <SpeechLanguageSelect model={{ languages: ["en", "de"], supportsLanguageDetection: true }} />,
+ );
+ await expect.element(picker).toHaveTextContent("Auto-detect (French unavailable)");
+ await screen.rerender(<SpeechLanguageSelect model={{ languages: ["en", "fr"] }} />);
+ await expect.element(picker).toHaveTextContent("French");
+ expect(getClientSettings().speechLanguage).toBe("fr");
+});
+
+it("keeps punctuation in a saved dictionary term and its alias", async () => {
+ const previous = getClientSettings();
+ using cleanup = new DisposableStack();
+ cleanup.defer(() => __setClientSettingsForTests(previous));
+ __setClientSettingsForTests({ ...previous, speechCustomWords: [] });
+ await using screen = await render(
+ <SettingsScopeProvider search={{}} onChange={() => {}}>
+ <VoiceSettingsPanel />
+ </SettingsScopeProvider>,
+ );
+ await userEvent.type(
+ screen.getByRole("textbox", { name: "Add a word or phrase to the dictionary" }),
+ "O'Reilly{Enter}",
+ );
+ expect(getClientSettings().speechCustomWords[0]?.term).toBe("O'Reilly");
+ await userEvent.click(screen.getByRole("button", { name: "Add an alias for O'Reilly" }));
+ await userEvent.type(
+ screen.getByRole("textbox", { name: "Alias for O'Reilly" }),
+ '"Oh Really"{Enter}',
+ );
+ expect(getClientSettings().speechCustomWords).toEqual([
+ { term: "O'Reilly", aliases: ['"Oh Really"'] },
+ ]);
+});
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx
new file mode 100644
index 00000000000..596e3ea295a
--- /dev/null
+++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx
@@ -0,0 +1,1371 @@
+import {
+ cancelEnvironmentSpeechModelDownload,
+ downloadEnvironmentSpeechModel,
+ getEnvironmentSpeechModels,
+ getEnvironmentSpeechStatus,
+ removeEnvironmentSpeechModel,
+ selectEnvironmentSpeechModel,
+ updateEnvironmentSpeechAcceleration,
+ updateEnvironmentSpeechModelUnloadTimeout,
+} from "@t3tools/client-runtime/voice-input";
+import type { PreparedConnection } from "@t3tools/client-runtime/connection";
+import {
+ AuthSettingsWriteScope,
+ DEFAULT_SPEECH_POST_PROCESSING_PROMPT,
+ type EnvironmentId,
+ type EnvironmentSpeechModel,
+ type EnvironmentSpeechStatus,
+ type ProjectId,
+ type SpeechAcceleration,
+ type SpeechCustomWords,
+ type SpeechModelUnloadTimeout,
+ type VoiceShortcutMode,
+} from "@t3tools/contracts";
+import { createModelSelection } from "@t3tools/shared/model";
+import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings";
+import {
+ AudioLinesIcon,
+ CheckIcon,
+ DownloadIcon,
+ LanguagesIcon,
+ MicIcon,
+ RefreshCwIcon,
+ StarIcon,
+ Trash2Icon,
+ XIcon,
+} from "lucide-react";
+import { useEffect, useRef, useState, type ReactNode } from "react";
+
+import { useStableCallback } from "~/hooks/useStableCallback";
+import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones";
+import {
+ chooseAudioDestination,
+ chooseTranscriptCleanup,
+ useCleanupGrant,
+} from "../../speech/voiceConsent";
+import { updateClientSettings, useClientSettings } from "../../hooks/useSettings";
+import { runtime } from "../../lib/runtime";
+import {
+ getCustomModelOptionsByInstance,
+ resolveAppModelSelectionState,
+} from "../../modelSelection";
+import { providerInstanceEntriesFor } from "../../providerInstances";
+import { useEnvironmentProviders, useEnvironments } from "../../state/environments";
+import { useEnvironmentScope } from "../../state/session";
+import { EMPTY_SERVER_PROVIDERS } from "../../state/server";
+import {
+ useEnvironmentSpeechInput,
+ useVoiceTranscriptionDestination,
+} from "../../speech/useEnvironmentSpeechInput";
+import { ComposerSpeechRecordingPill } from "../chat/ComposerSpeechButton";
+import { ProviderModelPicker } from "../chat/ProviderModelPicker";
+import { TraitsPicker } from "../chat/TraitsPicker";
+import { VoiceInputSetup } from "../chat/VoiceInputSetup";
+import { Badge } from "../ui/badge";
+import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
+import { Button } from "../ui/button";
+import { Input } from "../ui/input";
+import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
+import { Switch } from "../ui/switch";
+import { Textarea } from "../ui/textarea";
+import { toastManager } from "../ui/toast";
+import { MicrophoneTest } from "./MicrophoneTest";
+import { LanguageRow, ModelBenchmarks, orderSpeechLanguages } from "./SpeechModelComparison";
+import { languageLabel, SpeechLanguageSelect } from "./SpeechLanguageSelect";
+import { AdvancedSettings } from "./AdvancedSettings";
+import {
+ SETTINGS_PICKER_TRIGGER_CLASSNAME,
+ SettingResetButton,
+ SettingsPageContainer,
+ SettingsRow,
+ SettingsSearchTarget,
+ SettingsSection,
+} from "./settingsLayout";
+import { searchableSetting } from "./settingsSearch";
+import { useOptionalSettingsScope, useSettingsScope } from "./SettingsScopeContext";
+import { useScopedModelDisabledReason } from "./useScopedModelAvailability";
+import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings";
+
+const OFF = "off";
+const SYSTEM_DEFAULT = "system-default";
+const formatSize = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`;
+const SHORTCUT_MODE_LABELS: Record<VoiceShortcutMode, string> = {
+ auto: "Auto: press to toggle, hold to record",
+ hold: "Hold: record while held",
+ toggle: "Toggle: press to start, press again to stop",
+};
+const UNLOAD_LABELS: Record<SpeechModelUnloadTimeout, string> = {
+ never: "Never",
+ immediately: "Immediately",
+ min_2: "After 2 minutes",
+ min_5: "After 5 minutes",
+ min_10: "After 10 minutes",
+ min_15: "After 15 minutes",
+ hour_1: "After 1 hour",
+};
+const normalizeTerm = (value: string) => value.replace(/\s+/g, " ").trim().normalize("NFC");
+const spellingsOf = (words: SpeechCustomWords) =>
+ new Set(
+ words.flatMap(({ term, aliases }) =>
+ [term, ...aliases].map((spelling) => spelling.toLocaleLowerCase()),
+ ),
+ );
+const modelSortOrder = (model: EnvironmentSpeechModel) => {
+ if (model.active) return 0;
+ if (model.state === "installed") return 1;
+ if (model.state === "downloading" || model.state === "verifying") return 2;
+ return model.recommended ? 3 : 4;
+};
+
+type ModelCardProps = {
+ readonly model: EnvironmentSpeechModel;
+ readonly comparisonLanguages: readonly string[];
+ readonly busy: boolean;
+ readonly canCancel: boolean;
+ readonly onDownload: () => void;
+ readonly onSelect: () => void;
+ readonly onCancel: () => void;
+ readonly onDelete: () => void;
+ readonly testControls?: ReactNode;
+ readonly children?: ReactNode;
+};
+
+function ModelBadge(props: { icon: ReactNode; label: string; tooltip: string }) {
+ return (
+ <Tooltip>
+ <TooltipTrigger render={<Badge variant="outline" />}>
+ {props.icon}
+ {props.label}
+ </TooltipTrigger>
+ <TooltipPopup side="top">{props.tooltip}</TooltipPopup>
+ </Tooltip>
+ );
+}
+
+function ModelCard(props: ModelCardProps) {
+ const { model } = props;
+ const downloading = model.state === "downloading" || model.state === "verifying";
+ const progress = model.downloaded === undefined ? 0 : (model.downloaded / model.size) * 100;
+ return (
+ <div className="flex flex-wrap items-start gap-3 px-3 py-2.5 sm:px-4">
+ <div className="min-w-0 flex-1">
+ <div className="flex flex-wrap items-center gap-1.5">
+ <span className="text-sm font-medium leading-5">{model.name}</span>
+ {model.recommended ? (
+ <ModelBadge
+ icon={<StarIcon className="mr-1 size-3" />}
+ label="Recommended"
+ tooltip="A good default pick for the languages it covers."
+ />
+ ) : null}
+ {model.supportsStreaming ? (
+ <ModelBadge
+ icon={<AudioLinesIcon className="mr-1 size-3" />}
+ label="Live preview"
+ tooltip="Shows words in your draft as you speak, before the recording finishes."
+ />
+ ) : null}
+ {model.supportsTranslation ? (
+ <ModelBadge
+ icon={<LanguagesIcon className="mr-1 size-3" />}
+ label="Translate"
+ tooltip="Can write English text from speech in another language. Turn on Translate to English to use it."
+ />
+ ) : null}
+ </div>
+ <p className="mt-0.5 text-xs leading-relaxed text-muted-foreground">{model.description}</p>
+ <ModelBenchmarks
+ model={model}
+ languages={props.comparisonLanguages}
+ facts={[
+ ...(model.languages.length > 1
+ ? [`${model.languages.length} languages`]
+ : model.benchmark
+ ? []
+ : [languageLabel(model.languages[0] ?? "")]),
+ formatSize(model.size),
+ ]}
+ />
+ {downloading ? (
+ <div className="mt-2">
+ <div
+ role="progressbar"
+ aria-label={`${model.name} download progress`}
+ aria-valuemin={0}
+ aria-valuemax={100}
+ aria-valuenow={Math.round(progress)}
+ className="h-1 overflow-hidden rounded-full bg-muted"
+ >
+ <div
+ className="h-full bg-primary transition-[width] duration-200"
+ style={{ width: `${progress}%` }}
+ />
+ </div>
+ <p className="mt-1 text-2xs text-muted-foreground">
+ {model.state === "verifying"
+ ? "Verifying download…"
+ : `${Math.round(progress)}% downloaded`}
+ </p>
+ </div>
+ ) : null}
+ </div>
+ <div className="flex max-w-full flex-wrap items-center justify-end gap-1">
+ {props.testControls}
+ {downloading ? (
+ <Button
+ size="icon-sm"
+ variant="ghost"
+ aria-label={`Cancel ${model.name} download`}
+ disabled={!props.canCancel}
+ onClick={props.onCancel}
+ >
+ <XIcon className="size-3.5" />
+ </Button>
+ ) : model.state === "downloadable" ? (
+ <Button size="sm" variant="outline" disabled={props.busy} onClick={props.onDownload}>
+ <DownloadIcon className="mr-1.5 size-3.5" />
+ Download
+ </Button>
+ ) : model.active ? (
+ <Button size="sm" variant="secondary" disabled>
+ <CheckIcon className="mr-1.5 size-3.5" />
+ Selected
+ </Button>
+ ) : (
+ <Button size="sm" variant="outline" disabled={props.busy} onClick={props.onSelect}>
+ Use model
+ </Button>
+ )}
+ {model.state === "installed" ? (
+ <Button
+ size="icon-sm"
+ variant="ghost"
+ disabled={props.busy}
+ aria-label={`Delete ${model.name}`}
+ onClick={props.onDelete}
+ >
+ <Trash2Icon className="size-3.5" />
+ </Button>
+ ) : null}
+ </div>
+ {props.children}
+ </div>
+ );
+}
+
+function DictionaryEditor(props: {
+ readonly label: string;
+ readonly words: SpeechCustomWords;
+ readonly disabled?: boolean;
+ readonly onChange: (words: SpeechCustomWords) => void;
+}) {
+ const { words, label, disabled = false, onChange } = props;
+ const [termDraft, setTermDraft] = useState("");
+ const [aliasDraft, setAliasDraft] = useState<{ term: string; value: string } | null>(null);
+ const spellings = spellingsOf(words);
+ const seen = new Set(spellings);
+ const newTerms = termDraft
+ .split(/[,\n]/)
+ .map(normalizeTerm)
+ .filter((term) => {
+ const key = term.toLocaleLowerCase();
+ if (!key || seen.has(key)) return false;
+ seen.add(key);
+ return true;
+ });
+ const canAddTerm =
+ !disabled &&
+ newTerms.length > 0 &&
+ newTerms.every((term) => term.length <= 50) &&
+ words.length + newTerms.length <= 100;
+ const addTerm = () => {
+ if (!canAddTerm) return;
+ onChange([...words, ...newTerms.map((term) => ({ term, aliases: [] }))]);
+ setTermDraft("");
+ };
+ const addAlias = () => {
+ if (!aliasDraft) return;
+ const value = normalizeTerm(aliasDraft.value);
+ if (!value || value.length > 50 || spellings.has(value.toLocaleLowerCase())) return;
+ onChange(
+ words.map((entry) =>
+ entry.term === aliasDraft.term && entry.aliases.length < 8
+ ? { term: entry.term, aliases: [...entry.aliases, value] }
+ : entry,
+ ),
+ );
+ setAliasDraft(null);
+ };
+ return (
+ <>
+ <div className="flex items-start gap-1.5">
+ <Textarea
+ value={termDraft}
+ rows={1}
+ maxLength={5_100}
+ disabled={disabled}
+ placeholder="Add words or phrases, separated by commas or line breaks"
+ aria-label={`Add a word or phrase to the ${label}`}
+ onChange={(event) => setTermDraft(event.target.value)}
+ onKeyDown={(event) => {
+ if (event.key !== "Enter" || event.shiftKey) return;
+ event.preventDefault();
+ addTerm();
+ }}
+ />
+ <Button type="button" disabled={!canAddTerm} onClick={addTerm}>
+ Add
+ </Button>
+ </div>
+ {words.length > 0 ? (
+ <ul aria-label={`${label} entries`} className="mt-3 flex flex-wrap gap-1.5">
+ {words.map(({ term, aliases }) => (
+ <li
+ key={term}
+ className="flex items-center gap-1 rounded-md border border-border/60 bg-muted/30 py-0.5 pl-2 pr-0.5 text-xs"
+ >
+ <span className="font-medium">{term}</span>
+ {aliases.length > 0 ? (
+ <span className="text-muted-foreground">← {aliases.join(", ")}</span>
+ ) : null}
+ {aliasDraft?.term === term ? (
+ <Input
+ autoFocus
+ value={aliasDraft.value}
+ maxLength={50}
+ aria-label={`Alias for ${term}`}
+ placeholder="Misspelling"
+ className="w-32"
+ onChange={(event) => setAliasDraft({ term, value: event.target.value })}
+ onBlur={() => setAliasDraft(null)}
+ onKeyDown={(event) => {
+ if (event.key === "Escape") setAliasDraft(null);
+ if (event.key !== "Enter") return;
+ event.preventDefault();
+ addAlias();
+ }}
+ />
+ ) : (
+ <Button
+ type="button"
+ size="xs"
+ variant="ghost-muted"
+ disabled={disabled || aliases.length >= 8}
+ aria-label={`Add an alias for ${term}`}
+ onClick={() => setAliasDraft({ term, value: "" })}
+ >
+ Alias
+ </Button>
+ )}
+ <Button
+ type="button"
+ size="icon-xs"
+ variant="ghost-muted"
+ disabled={disabled}
+ aria-label={`Remove ${term}`}
+ onClick={() => onChange(words.filter((entry) => entry.term !== term))}
+ >
+ <XIcon />
+ </Button>
+ </li>
+ ))}
+ </ul>
+ ) : null}
+ </>
+ );
+}
+
+/** Vocabulary stored on the environment with the project, read by everyone who dictates there. */
+function ProjectDictionaryRow() {
+ const scope = useOptionalSettingsScope();
+ const words = useScopedSettings((settings) => settings.speechProjectCustomWords);
+ const updateScopedSettings = useUpdateScopedSettings();
+ if (scope === null) return null;
+ const isProjectScope = scope.scope.kind === "project" || scope.scope.kind === "checkout";
+ return (
+ <SettingsRow
+ serverScoped
+ settingKeys={["speechProjectCustomWords"]}
+ {...searchableSetting("speech-project-dictionary")}
+ title={isProjectScope ? "Project dictionary" : "Shared dictionary"}
+ description={
+ isProjectScope
+ ? `Saved with ${scope.scope.label} on its environment. Everyone who dictates in this project gets these spellings, along with their own dictionary. Project spellings win when the two overlap.`
+ : "Saved on the environment and used in every project there, along with each person’s own dictionary. Pick a project above to limit these words to one project."
+ }
+ control={
+ <span className="text-xs text-muted-foreground">
+ {words.length} {words.length === 1 ? "entry" : "entries"}
+ </span>
+ }
+ >
+ <div className="mt-2 pb-3.5">
+ <DictionaryEditor
+ label={isProjectScope ? "project dictionary" : "shared dictionary"}
+ words={words}
+ disabled={scope.connectedEnvironments.length === 0}
+ onChange={(next) => updateScopedSettings({ speechProjectCustomWords: next })}
+ />
+ </div>
+ </SettingsRow>
+ );
+}
+
+/** Cleanup runs on the draft's environment, so its model is an environment setting there. */
+function CleanupModelRow() {
+ const { environment, connectedEnvironments } = useSettingsScope();
+ const settings = useScopedSettings();
+ const updateSettings = useUpdateScopedSettings();
+ const providersByEnvironment = useEnvironmentProviders();
+ const providers =
+ (environment?.serverConfig
+ ? providersByEnvironment.get(environment.environmentId)
+ : undefined) ?? EMPTY_SERVER_PROVIDERS;
+ const textGenerationProviders = providers.filter(
+ (provider) => provider.supportsTextGeneration !== false,
+ );
+ const instanceEntries = providerInstanceEntriesFor(textGenerationProviders, settings, null);
+ const hasProvider = instanceEntries.some((entry) => entry.enabled && entry.isAvailable);
+ const dedicated = settings.speechPostProcessingModelSelection !== null;
+ const selection = resolveAppModelSelectionState(
+ {
+ ...settings,
+ textGenerationModelSelection: resolveSpeechPostProcessingModelSelection(
+ settings,
+ textGenerationProviders,
+ ),
+ },
+ textGenerationProviders,
+ );
+ const modelOptionsByInstance = getCustomModelOptionsByInstance(
+ settings,
+ textGenerationProviders,
+ selection.instanceId,
+ selection.model,
+ );
+ const instanceEntry = instanceEntries.find((entry) => entry.instanceId === selection.instanceId);
+ const modelDisabledReason = useScopedModelDisabledReason(settings, instanceEntries);
+ const setModel = (next: ReturnType<typeof createModelSelection>) => {
+ const reason = modelDisabledReason(next.instanceId, next.model);
+ if (reason) {
+ toastManager.add({ type: "error", title: "Cleanup model not saved", description: reason });
+ return;
+ }
+ updateSettings({ speechPostProcessingModelSelection: next });
+ };
+ return (
+ <SettingsRow
+ serverScoped
+ settingKeys={["speechPostProcessingModelSelection"]}
+ {...searchableSetting("speech-post-processing-model")}
+ title="Cleanup model"
+ description={
+ dedicated
+ ? `Drafts on ${environment?.label ?? "this environment"} use this model for cleanup instead of the environment’s text generation model.`
+ : `Cleanup currently uses the text generation model on ${environment?.label ?? "this environment"}. Pick another model if you want a cheaper or faster one.`
+ }
+ resetAction={
+ dedicated ? (
+ <SettingResetButton
+ label="cleanup model"
+ onClick={() => updateSettings({ speechPostProcessingModelSelection: null })}
+ />
+ ) : null
+ }
+ control={
+ connectedEnvironments.length === 0 ? (
+ <span className="text-sm text-muted-foreground">Connect an environment first.</span>
+ ) : !hasProvider || !instanceEntry ? (
+ <span className="text-sm text-muted-foreground">No providers available.</span>
+ ) : (
+ <div className="flex min-w-0 flex-wrap items-center justify-end gap-1.5">
+ <ProviderModelPicker
+ environmentId={environment?.environmentId ?? null}
+ activeInstanceId={selection.instanceId}
+ model={selection.model}
+ lockedProvider={null}
+ instanceEntries={instanceEntries}
+ modelOptionsByInstance={modelOptionsByInstance}
+ triggerClassName={SETTINGS_PICKER_TRIGGER_CLASSNAME}
+ getModelDisabledReason={modelDisabledReason}
+ onInstanceModelChange={(instanceId, model) =>
+ setModel(createModelSelection(instanceId, model))
+ }
+ />
+ <TraitsPicker
+ provider={instanceEntry.driverKind}
+ models={instanceEntry.models}
+ model={selection.model}
+ modelOptions={selection.options ?? []}
+ allowPromptInjectedEffort={false}
+ planModeEnabled={settings.planModeEnabled}
+ triggerClassName={SETTINGS_PICKER_TRIGGER_CLASSNAME}
+ onModelOptionsChange={(options) =>
+ setModel(createModelSelection(selection.instanceId, selection.model, options))
+ }
+ />
+ </div>
+ )
+ }
+ />
+ );
+}
+
+function useVoiceModels(prepared: PreparedConnection | null) {
+ const canManage = useEnvironmentScope(prepared?.environmentId ?? null, AuthSettingsWriteScope);
+ const [models, setModels] = useState<readonly EnvironmentSpeechModel[]>([]);
+ const [status, setStatus] = useState<EnvironmentSpeechStatus | null>(null);
+ const [busy, setBusy] = useState(false);
+ const [requestedDownload, setRequestedDownload] = useState<string | null>(null);
+ const [actionConnection, setActionConnection] = useState(prepared);
+ if (actionConnection !== prepared) {
+ setActionConnection(prepared);
+ setBusy(false);
+ setRequestedDownload(null);
+ setModels([]);
+ setStatus(null);
+ }
+ const epoch = useRef(0);
+ const refreshRevision = useRef(0);
+ const refresh = useStableCallback(
+ async (connection: PreparedConnection, mode: "foreground" | "progress") => {
+ const current = epoch.current;
+ const revision = ++refreshRevision.current;
+ try {
+ const [nextStatus, nextModels] = await Promise.all([
+ runtime.runPromise(getEnvironmentSpeechStatus(connection)),
+ runtime.runPromise(getEnvironmentSpeechModels(connection, mode === "foreground")),
+ ]);
+ if (current !== epoch.current || revision !== refreshRevision.current) return;
+ setStatus(nextStatus);
+ setModels((previous) => {
+ if (mode === "foreground") return nextModels.models;
+ const benchmarks = new Map(previous.map((model) => [model.id, model.benchmark]));
+ return nextModels.models.map((model) => {
+ const benchmark = benchmarks.get(model.id);
+ return benchmark ? { ...model, benchmark } : model;
+ });
+ });
+ } catch (error) {
+ if (mode === "progress") return;
+ if (current !== epoch.current || revision !== refreshRevision.current) return;
+ toastManager.add({
+ type: "error",
+ title: "Could not load transcription models",
+ description: error instanceof Error ? error.message : String(error),
+ });
+ }
+ },
+ );
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- loads the environment's model catalog when its connection becomes available
+ useEffect(() => {
+ epoch.current += 1;
+ if (prepared) void refresh(prepared, "foreground");
+ return () => {
+ epoch.current += 1;
+ };
+ }, [prepared, refresh]);
+ const downloading =
+ requestedDownload !== null ||
+ models.some((model) => model.state === "downloading" || model.state === "verifying");
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls progress while a local request or the environment reports an active download
+ useEffect(() => {
+ if (!prepared || !downloading) return;
+ let cancelled = false;
+ let timer: number | undefined;
+ const poll = async () => {
+ await refresh(prepared, "progress");
+ if (!cancelled) timer = window.setTimeout(() => void poll(), 1_000);
+ };
+ void poll();
+ return () => {
+ cancelled = true;
+ window.clearTimeout(timer);
+ };
+ }, [downloading, prepared, refresh]);
+
+ const run = async (label: string, work: () => Promise<unknown>, downloadId?: string) => {
+ if (!prepared || !canManage) return;
+ const current = epoch.current;
+ if (downloadId) setRequestedDownload(downloadId);
+ else setBusy(true);
+ try {
+ await work();
+ } catch (error) {
+ if (current === epoch.current)
+ toastManager.add({
+ type: "error",
+ title: label,
+ description: error instanceof Error ? error.message : String(error),
+ });
+ }
+ if (current === epoch.current) {
+ if (downloadId) setRequestedDownload(null);
+ else setBusy(false);
+ await refresh(prepared, "foreground");
+ }
+ };
+ return {
+ models,
+ status,
+ run,
+ controlsDisabled: busy || downloading || !canManage,
+ canCancel: canManage && !busy,
+ requestedDownload,
+ };
+}
+
+type VoiceModels = ReturnType<typeof useVoiceModels>;
+
+function AdvancedSpeechSettings({
+ prepared,
+ catalog,
+}: {
+ prepared: PreparedConnection;
+ catalog: VoiceModels;
+}) {
+ const { run, controlsDisabled, status } = catalog;
+ if (!status?.supported) return null;
+ const supported = status;
+ return (
+ <AdvancedSettings targetIds={[searchableSetting("speech-hardware").id]}>
+ <SettingsRow
+ {...searchableSetting("speech-hardware")}
+ title="Hardware"
+ description="Auto uses the environment’s GPU when it has one."
+ control={
+ <Select
+ value={supported.acceleration}
+ disabled={controlsDisabled}
+ onValueChange={(value) => {
+ if (value)
+ void run("Could not change transcription hardware", () =>
+ runtime.runPromise(
+ updateEnvironmentSpeechAcceleration(prepared, value as SpeechAcceleration),
+ ),
+ );
+ }}
+ >
+ <SelectTrigger size="sm" aria-label="Transcription hardware" className="max-w-72">
+ <SelectValue>
+ {supported.acceleration === "auto"
+ ? "Auto"
+ : supported.acceleration === "cpu"
+ ? "CPU"
+ : (supported.gpuDevices.find(
+ (device) => `gpu:${device.id}` === supported.acceleration,
+ )?.name ?? "Selected GPU")}
+ </SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ <SelectItem value="auto">Auto</SelectItem>
+ <SelectItem value="cpu">CPU</SelectItem>
+ {supported.gpuDevices.map((device) => (
+ <SelectItem key={device.id} value={`gpu:${device.id}`}>
+ {device.name}
+ </SelectItem>
+ ))}
+ </SelectPopup>
+ </Select>
+ }
+ />
+ <SettingsRow
+ title="Unload model"
+ description="Release the model’s memory on the environment after it sits idle."
+ control={
+ <Select
+ value={supported.modelUnloadTimeout}
+ disabled={controlsDisabled}
+ onValueChange={(value) => {
+ if (value)
+ void run("Could not change when the model unloads", () =>
+ runtime.runPromise(
+ updateEnvironmentSpeechModelUnloadTimeout(
+ prepared,
+ value as SpeechModelUnloadTimeout,
+ ),
+ ),
+ );
+ }}
+ >
+ <SelectTrigger size="sm" aria-label="Unload model" className="max-w-72">
+ <SelectValue>{UNLOAD_LABELS[supported.modelUnloadTimeout]}</SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ {(Object.keys(UNLOAD_LABELS) as SpeechModelUnloadTimeout[]).map((timeout) => (
+ <SelectItem key={timeout} value={timeout}>
+ {UNLOAD_LABELS[timeout]}
+ </SelectItem>
+ ))}
+ </SelectPopup>
+ </Select>
+ }
+ />
+ </AdvancedSettings>
+ );
+}
+
+function SpeechModelBrowser(props: {
+ prepared: PreparedConnection;
+ catalog: VoiceModels;
+ testEnvironmentId: EnvironmentId;
+ projectId: ProjectId | null;
+ projectName: string | null;
+}) {
+ const {
+ prepared,
+ catalog: { status, models, run, controlsDisabled, canCancel, requestedDownload },
+ } = props;
+ const [search, setSearch] = useState("");
+ const [languageSearch, setLanguageSearch] = useState("");
+ const [selection, setSelection] = useState<string[] | null>(null);
+ const preferred = navigator.languages.map((language) => language.toLowerCase().split("-")[0]!);
+ const supportedLanguages = [...new Set(models.flatMap((model) => model.languages))];
+ const defaultLanguage = preferred.find((code) => supportedLanguages.includes(code));
+ const selectedLanguages = selection ?? (defaultLanguage ? [defaultLanguage] : []);
+ const languages = orderSpeechLanguages(supportedLanguages, preferred);
+ const needle = languageSearch.trim().toLocaleLowerCase();
+ const visibleLanguages = languages.filter(
+ (code) =>
+ !needle || code.includes(needle) || languageLabel(code).toLocaleLowerCase().includes(needle),
+ );
+ const query = search.trim().toLocaleLowerCase();
+ const sortedModels = models
+ .filter(
+ (model) =>
+ selectedLanguages.every((language) => model.languages.includes(language)) &&
+ (!query || (model.name + " " + model.description).toLocaleLowerCase().includes(query)),
+ )
+ .toSorted((left, right) => modelSortOrder(left) - modelSortOrder(right));
+ const activeModel = models.find((model) => model.active);
+ const renderedModels =
+ activeModel && !sortedModels.includes(activeModel)
+ ? [activeModel, ...sortedModels]
+ : sortedModels;
+
+ return (
+ <SettingsSearchTarget
+ {...searchableSetting("speech-models")}
+ className="@container/voice-models"
+ >
+ {status && !status.supported ? (
+ <p className="px-4 py-2 text-sm text-muted-foreground">
+ This environment can’t transcribe audio: {status.reason}.
+ </p>
+ ) : null}
+ {status?.supported ? (
+ <>
+ <p className="border-b border-border/60 px-3 py-2 text-xs text-muted-foreground sm:px-4">
+ Pick languages to see models that support all of them. Accuracy is word or character
+ error rate: lower is better. Speed is the time to transcribe 30 seconds of audio on
+ Handy’s benchmark GPU.
+ </p>
+ <div className="@min-[32rem]/voice-models:grid @min-[32rem]/voice-models:grid-cols-[11rem_minmax(0,1fr)]">
+ <div className="flex min-h-0 flex-col border-b border-border/60 bg-muted/10 @min-[32rem]/voice-models:border-r @min-[32rem]/voice-models:border-b-0">
+ <div className="space-y-2 p-3">
+ <p className="text-sm font-medium">Languages</p>
+ <Input
+ size="compact"
+ aria-label="Search languages"
+ placeholder="Search languages"
+ value={languageSearch}
+ onChange={(e) => setLanguageSearch(e.target.value)}
+ />
+ </div>
+ <div
+ role="group"
+ aria-label="Browse models by language"
+ className="flex gap-1 overflow-x-auto px-2 pb-2 @min-[32rem]/voice-models:h-72 @min-[32rem]/voice-models:flex-col @min-[32rem]/voice-models:overflow-y-auto"
+ >
+ <LanguageRow
+ label="All languages"
+ selected={selectedLanguages.length === 0}
+ onClick={() => setSelection([])}
+ />
+ {visibleLanguages.map((code) => (
+ <LanguageRow
+ key={code}
+ label={languageLabel(code)}
+ selected={selectedLanguages.includes(code)}
+ onClick={() =>
+ setSelection(
+ selectedLanguages.includes(code)
+ ? selectedLanguages.filter((language) => language !== code)
+ : [...selectedLanguages, code],
+ )
+ }
+ />
+ ))}
+ {visibleLanguages.length === 0 && (
+ <p className="px-2 py-1 text-xs text-muted-foreground">No languages match.</p>
+ )}
+ </div>
+ </div>
+ <div className="flex min-w-0 flex-col">
+ <div className="space-y-2 p-3">
+ <p className="text-sm font-medium">
+ {selectedLanguages.length
+ ? "Models for " + selectedLanguages.map(languageLabel).join(", ")
+ : "All models"}
+ </p>
+ <Input
+ size="compact"
+ aria-label="Search transcription models"
+ placeholder="Search models"
+ value={search}
+ onChange={(e) => setSearch(e.target.value)}
+ />
+ </div>
+ <div
+ className="h-72 overflow-y-auto divide-y divide-border/60"
+ role="region"
+ aria-label="Transcription models"
+ tabIndex={0}
+ >
+ {sortedModels.length === 0 && (
+ <p className="px-4 py-2 text-sm text-muted-foreground">No models match.</p>
+ )}
+ {renderedModels.map((model) => {
+ const card: ModelCardProps = {
+ model:
+ model.id === requestedDownload && model.state === "downloadable"
+ ? { ...model, state: "downloading" }
+ : model,
+ busy: controlsDisabled,
+ canCancel,
+ comparisonLanguages: selectedLanguages,
+ onDownload: () =>
+ void run(
+ "Could not download the model",
+ () =>
+ runtime.runPromise(downloadEnvironmentSpeechModel(prepared, model.id)),
+ model.id,
+ ),
+ onSelect: () =>
+ void run("Could not select the model", () =>
+ runtime.runPromise(selectEnvironmentSpeechModel(prepared, model.id)),
+ ),
+ onCancel: () =>
+ void run("Could not cancel the download", () =>
+ runtime.runPromise(
+ cancelEnvironmentSpeechModelDownload(prepared, model.id),
+ ),
+ ),
+ onDelete: () =>
+ void run("Could not delete the model", () =>
+ runtime.runPromise(removeEnvironmentSpeechModel(prepared, model.id)),
+ ),
+ };
+ return model.active ? (
+ <TranscriptionTest
+ key={model.id}
+ card={card}
+ hidden={!sortedModels.includes(model)}
+ environmentId={props.testEnvironmentId}
+ projectId={props.projectId}
+ projectName={props.projectName}
+ />
+ ) : (
+ <ModelCard key={model.id} {...card} />
+ );
+ })}
+ </div>
+ </div>
+ </div>
+ </>
+ ) : null}
+ </SettingsSearchTarget>
+ );
+}
+
+function TranscriptionTest(props: {
+ readonly card: ModelCardProps;
+ readonly hidden: boolean;
+ readonly environmentId: EnvironmentId;
+ readonly projectId: ProjectId | null;
+ readonly projectName: string | null;
+}) {
+ const [transcript, setTranscript] = useState<string | null>(null);
+ const draft = transcript ?? "";
+ const [showTranscript, setShowTranscript] = useState(false);
+ const draftRef = useRef(draft);
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes the committed draft to the asynchronous transcription controller
+ useEffect(() => {
+ draftRef.current = draft;
+ }, [draft]);
+ const speech = useEnvironmentSpeechInput({
+ environmentId: props.environmentId,
+ projectId: props.projectId,
+ projectName: props.projectName,
+ ownerKey: JSON.stringify([props.environmentId, props.projectId]),
+ draftText: draft,
+ cleanup: false,
+ readDraft: () => ({
+ text: draftRef.current,
+ selection: { start: draftRef.current.length, end: draftRef.current.length },
+ }),
+ commitDraft: (text) => setTranscript(text),
+ });
+ return (
+ <SettingsSearchTarget {...searchableSetting("transcription-test")} hidden={props.hidden}>
+ <ModelCard
+ {...props.card}
+ testControls={
+ <div className="flex min-w-0 max-w-full items-center gap-1">
+ <ComposerSpeechRecordingPill
+ state={speech.state}
+ progress={null}
+ level={speech.level}
+ onStart={() => void speech.start()}
+ onStop={() => void speech.stop()}
+ onCancel={() => speech.cancel()}
+ onSkipPostProcessing={() => speech.skipPostProcessing()}
+ />
+ {speech.available ? (
+ speech.state.phase === "idle" ? (
+ <Button
+ size="sm"
+ variant="outline"
+ onClick={() => {
+ setShowTranscript(true);
+ void speech.start();
+ }}
+ >
+ <MicIcon />
+ Test model
+ </Button>
+ ) : null
+ ) : (
+ <span className="text-xs text-muted-foreground">
+ {speech.status?.supported === false
+ ? speech.status.reason
+ : "Waiting for the environment to connect"}
+ </span>
+ )}
+ </div>
+ }
+ >
+ {showTranscript ? (
+ <div className="w-full">
+ <div className="flex items-start gap-2">
+ <p className="min-w-0 flex-1 text-xs leading-relaxed text-muted-foreground">
+ Say a sentence in the language above. Cleanup is skipped, so you see exactly what
+ the model heard. The transcript stays on this page.
+ </p>
+ <Button
+ size="icon-sm"
+ variant="ghost"
+ aria-label="Close model test"
+ onClick={() => {
+ speech.cancel();
+ setShowTranscript(false);
+ }}
+ >
+ <XIcon />
+ </Button>
+ </div>
+ {speech.preview || transcript !== null ? (
+ <p
+ role="status"
+ aria-label="Test transcript"
+ className="mt-2 select-text whitespace-pre-wrap break-words text-sm leading-relaxed"
+ >
+ {speech.preview ? (
+ <>
+ {speech.preview.committed}
+ <span className="text-muted-foreground">{speech.preview.tentative}</span>
+ </>
+ ) : (
+ transcript || "No speech recognized. Try again."
+ )}
+ </p>
+ ) : null}
+ </div>
+ ) : null}
+ </ModelCard>
+ <VoiceInputSetup
+ environmentId={speech.transcriptionEnvironmentId}
+ open={speech.setup.open}
+ step={speech.setup.step}
+ status={speech.status}
+ model={speech.setup.model}
+ downloading={speech.setup.downloading}
+ error={speech.setup.error}
+ onOpenChange={speech.setup.setOpen}
+ onDownload={() => void speech.setup.download()}
+ onCancelDownload={() => void speech.setup.cancelDownload()}
+ onStartRecording={() => void speech.setup.startRecording()}
+ />
+ </SettingsSearchTarget>
+ );
+}
+
+export function VoiceSettingsPanel() {
+ const cleanupEnabled = useCleanupGrant() !== null;
+ const settings = useClientSettings();
+ const settingsScope = useOptionalSettingsScope();
+ const destination = useVoiceTranscriptionDestination();
+ const { environments } = useEnvironments();
+ const [microphones, setMicrophones] = useState<BrowserMicrophone[]>([]);
+ const prepared = destination.prepared;
+ const catalog = useVoiceModels(prepared);
+
+ const refreshMicrophones = useStableCallback(async () => {
+ if (!navigator.mediaDevices?.enumerateDevices) return;
+ try {
+ setMicrophones(await listBrowserMicrophones());
+ } catch {
+ setMicrophones([]);
+ }
+ });
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to microphone device changes while settings are open
+ useEffect(() => {
+ queueMicrotask(() => void refreshMicrophones());
+ const mediaDevices = navigator.mediaDevices;
+ if (!mediaDevices?.addEventListener) return;
+ const onChange = () => void refreshMicrophones();
+ mediaDevices.addEventListener("devicechange", onChange);
+ return () => mediaDevices.removeEventListener("devicechange", onChange);
+ }, [refreshMicrophones]);
+ const modelStatus = catalog.status;
+ const target = settingsScope?.target;
+ const members = settingsScope?.scope.members;
+ const project =
+ members?.find(
+ (member) => member.environmentId === target?.environmentId && member.id === target.projectId,
+ ) ?? members?.[0];
+
+ const selectedMicrophone = settings.voiceMicrophone;
+ const selectedMicrophoneLabel = selectedMicrophone
+ ? (microphones.find((device) => device.deviceId === selectedMicrophone)?.label ??
+ "Selected microphone (unavailable)")
+ : "System default";
+ const destinationValue = destination.environmentId ?? OFF;
+ const chosenEnvironment = environments.find(
+ (environment) => environment.environmentId === destination.environmentId,
+ );
+ const chosenUnsupported =
+ chosenEnvironment?.serverConfig != null &&
+ chosenEnvironment.serverConfig.environment.capabilities.voiceTranscription !== true;
+ const model = prepared && modelStatus?.supported ? modelStatus : null;
+
+ return (
+ <SettingsPageContainer
+ searchTargetFallbacks={{
+ "speech-models": prepared ? undefined : "voice-transcription-destination",
+ "speech-hardware": prepared && model ? undefined : "voice-transcription-destination",
+ "transcription-test": !prepared
+ ? "voice-transcription-destination"
+ : !model || model.state === "missing-model"
+ ? "speech-models"
+ : undefined,
+ "speech-post-processing-model":
+ cleanupEnabled && settingsScope ? undefined : "speech-post-processing",
+ "speech-correction-word": cleanupEnabled ? undefined : "speech-post-processing",
+ }}
+ >
+ <SettingsSection id="voice-input" title="Input">
+ <SettingsRow
+ {...searchableSetting("voice-microphone")}
+ title="Microphone"
+ description="System default follows your operating system. Record a ten-second test and play it back here. Test audio stays on this device and is never transcribed."
+ control={
+ <div className="flex min-w-0 flex-wrap items-center justify-end gap-1.5">
+ <SettingsSearchTarget {...searchableSetting("microphone-test")}>
+ <MicrophoneTest key={selectedMicrophone} microphoneId={selectedMicrophone} />
+ </SettingsSearchTarget>
+ <div className="flex min-w-0 items-center gap-1.5">
+ <Select
+ value={selectedMicrophone ? `device:${selectedMicrophone}` : SYSTEM_DEFAULT}
+ onValueChange={(value) => {
+ if (value)
+ void updateClientSettings({
+ voiceMicrophone:
+ value === SYSTEM_DEFAULT ? "" : value.slice("device:".length),
+ });
+ }}
+ >
+ <SelectTrigger size="sm" aria-label="Microphone" className="max-w-64">
+ <SelectValue>{selectedMicrophoneLabel}</SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ <SelectItem value={SYSTEM_DEFAULT}>System default</SelectItem>
+ {microphones.map((device) => (
+ <SelectItem key={device.deviceId} value={`device:${device.deviceId}`}>
+ {device.label}
+ </SelectItem>
+ ))}
+ </SelectPopup>
+ </Select>
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon-sm"
+ aria-label="Refresh microphones"
+ onClick={() => void refreshMicrophones()}
+ >
+ <RefreshCwIcon className="size-3.5" />
+ </Button>
+ </div>
+ </div>
+ }
+ />
+ <SettingsRow
+ {...searchableSetting("dictation-shortcut-mode")}
+ title="Dictation shortcut"
+ description="How the dictation shortcut starts and stops recording."
+ control={
+ <Select
+ value={settings.voiceShortcutMode}
+ onValueChange={(value) => {
+ if (value)
+ void updateClientSettings({ voiceShortcutMode: value as VoiceShortcutMode });
+ }}
+ >
+ <SelectTrigger size="sm" aria-label="Dictation shortcut mode" className="max-w-72">
+ <SelectValue>{SHORTCUT_MODE_LABELS[settings.voiceShortcutMode]}</SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ {(Object.keys(SHORTCUT_MODE_LABELS) as VoiceShortcutMode[]).map((mode) => (
+ <SelectItem key={mode} value={mode}>
+ {SHORTCUT_MODE_LABELS[mode]}
+ </SelectItem>
+ ))}
+ </SelectPopup>
+ </Select>
+ }
+ />
+ </SettingsSection>
+
+ <SettingsSection id="voice-transcription" title="Transcription">
+ <SettingsRow
+ {...searchableSetting("voice-transcription-destination")}
+ title="Send microphone audio to"
+ description={
+ <>
+ Microphone audio is sent to this environment to transcribe, then discarded. Choose Off
+ to turn voice input off on this device. This choice is saved on this device only.
+ </>
+ }
+ control={
+ <Select
+ value={destinationValue}
+ onValueChange={(value) => {
+ if (!value) return;
+ chooseAudioDestination(
+ value === OFF
+ ? null
+ : { kind: "environment", environmentId: value as EnvironmentId },
+ );
+ }}
+ >
+ <SelectTrigger size="sm" aria-label="Send microphone audio to" className="max-w-72">
+ <SelectValue>
+ {destinationValue === OFF
+ ? "Off"
+ : (chosenEnvironment?.label ?? "Chosen environment (unavailable)")}
+ </SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ <SelectItem value={OFF}>Off</SelectItem>
+ {environments.map((environment) => (
+ <SelectItem key={environment.environmentId} value={environment.environmentId}>
+ {environment.label}
+ </SelectItem>
+ ))}
+ {destination.environmentId && !chosenEnvironment ? (
+ <SelectItem value={destination.environmentId}>
+ Chosen environment (unavailable)
+ </SelectItem>
+ ) : null}
+ </SelectPopup>
+ </Select>
+ }
+ >
+ {destination.environmentId && chosenUnsupported ? (
+ <p className="mt-2 text-xs text-destructive">
+ {chosenEnvironment.label} can’t transcribe audio. Pick an environment that can to use
+ voice input.
+ </p>
+ ) : null}
+ </SettingsRow>
+
+ <SettingsRow
+ {...searchableSetting("speech-language")}
+ title="Language"
+ description={
+ model
+ ? `${model.model} ${model.supportsLanguageDetection ? "detects the language automatically" : "uses one fixed language"}.`
+ : "Choose an environment above to see the languages its transcription model supports."
+ }
+ control={<SpeechLanguageSelect model={model} />}
+ />
+ {model?.supportsTranslation ? (
+ <SettingsRow
+ title="Translate to English"
+ description="Speak any language and get English text."
+ control={
+ <Switch
+ aria-label="Translate to English"
+ checked={settings.speechTranslateToEnglish}
+ onCheckedChange={(checked) =>
+ void updateClientSettings({ speechTranslateToEnglish: checked })
+ }
+ />
+ }
+ />
+ ) : null}
+ {prepared ? (
+ <SpeechModelBrowser
+ key={prepared.environmentId}
+ prepared={prepared}
+ catalog={catalog}
+ testEnvironmentId={
+ project?.environmentId ?? target?.environmentId ?? prepared.environmentId
+ }
+ projectId={project?.id ?? null}
+ projectName={project?.title ?? null}
+ />
+ ) : null}
+ {prepared && <AdvancedSpeechSettings prepared={prepared} catalog={catalog} />}
+ </SettingsSection>
+ <SettingsSection id="voice-recognition" title="Transcription options">
+ <SettingsRow
+ {...searchableSetting("speech-silence-filtering")}
+ title="Silence filtering"
+ description="Skip silence and other non-speech audio before transcription, keeping a short buffer so words aren’t cut off."
+ control={
+ <Switch
+ aria-label="Silence filtering"
+ checked={settings.speechSilenceFiltering}
+ onCheckedChange={(checked) =>
+ void updateClientSettings({ speechSilenceFiltering: checked })
+ }
+ />
+ }
+ />
+ <SettingsRow
+ {...searchableSetting("remove-filler-words")}
+ title="Remove filler words"
+ description="Remove filler words like “um” and “uh” from transcripts."
+ control={
+ <Switch
+ aria-label="Remove filler words"
+ checked={settings.speechRemoveFillerWords}
+ onCheckedChange={(checked) =>
+ void updateClientSettings({ speechRemoveFillerWords: checked })
+ }
+ />
+ }
+ />
+ <SettingsRow
+ {...searchableSetting("speech-dictionary")}
+ title="Dictionary"
+ description="Names and terms transcription should spell your way, saved on this device. Add an alias when it keeps hearing a term wrong. The name of the project you dictate in is also sent as a hint."
+ control={
+ <span className="text-xs text-muted-foreground">
+ {settings.speechCustomWords.length}{" "}
+ {settings.speechCustomWords.length === 1 ? "entry" : "entries"}
+ </span>
+ }
+ >
+ <div className="mt-2 pb-3.5">
+ <DictionaryEditor
+ label="dictionary"
+ words={settings.speechCustomWords}
+ onChange={(words) => void updateClientSettings({ speechCustomWords: words })}
+ />
+ </div>
+ </SettingsRow>
+ <ProjectDictionaryRow />
+ </SettingsSection>
+
+ <SettingsSection id={searchableSetting("speech-post-processing").id} title="Cleanup">
+ <SettingsRow
+ title="Clean up transcripts with a provider"
+ description="Sends each transcript, plus the draft text around your cursor, to the text generation provider on the draft’s environment to fix punctuation, numbers, and spoken corrections. Your audio is never sent. Off by default."
+ control={
+ <Switch
+ aria-label="Clean up transcripts with a provider"
+ checked={cleanupEnabled}
+ onCheckedChange={chooseTranscriptCleanup}
+ />
+ }
+ />
+ {cleanupEnabled ? (
+ <SettingsRow
+ title="Cleanup instructions"
+ description="Use the built-in instructions or write your own."
+ control={
+ <Select
+ value={settings.speechPostProcessingPrompt.mode}
+ onValueChange={(value) => {
+ if (value === "default" || value === "custom")
+ void updateClientSettings({
+ speechPostProcessingPrompt: {
+ ...settings.speechPostProcessingPrompt,
+ mode: value,
+ },
+ });
+ }}
+ >
+ <SelectTrigger size="sm" aria-label="Cleanup instructions" className="max-w-64">
+ <SelectValue>
+ {settings.speechPostProcessingPrompt.mode === "custom" ? "Custom" : "Built-in"}
+ </SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ <SelectItem value="default">Built-in</SelectItem>
+ <SelectItem value="custom">Custom</SelectItem>
+ </SelectPopup>
+ </Select>
+ }
+ >
+ <Textarea
+ aria-label="Custom cleanup instructions"
+ className="mt-2"
+ maxLength={10_000}
+ disabled={settings.speechPostProcessingPrompt.mode !== "custom"}
+ value={
+ settings.speechPostProcessingPrompt.mode === "custom"
+ ? settings.speechPostProcessingPrompt.customInstructions
+ : DEFAULT_SPEECH_POST_PROCESSING_PROMPT
+ }
+ onChange={(event) =>
+ void updateClientSettings({
+ speechPostProcessingPrompt: {
+ mode: "custom",
+ customInstructions: event.target.value,
+ },
+ })
+ }
+ />
+ </SettingsRow>
+ ) : null}
+ {cleanupEnabled && settingsScope !== null ? <CleanupModelRow /> : null}
+ {cleanupEnabled ? (
+ <SettingsRow
+ {...searchableSetting("speech-correction-word")}
+ title="Correction cue"
+ description="The word or phrase you say to flag a correction, like “sorry” or “scratch that”. Cleanup applies the correction only where one is meant, and hears the cue as a hint."
+ control={
+ <Input
+ key={settings.speechCorrectionWord}
+ defaultValue={settings.speechCorrectionWord}
+ maxLength={50}
+ placeholder="sorry"
+ aria-label="Correction cue"
+ className="w-40"
+ onBlur={(event) => {
+ const next = event.target.value.trim();
+ if (next !== settings.speechCorrectionWord)
+ void updateClientSettings({ speechCorrectionWord: next });
+ }}
+ />
+ }
+ />
+ ) : null}
+ </SettingsSection>
+ </SettingsPageContainer>
+ );
+}
diff --git a/apps/web/src/components/settings/settingsLayout.tsx b/apps/web/src/components/settings/settingsLayout.tsx
index cf2aae4cf53..605dd377397 100644
--- a/apps/web/src/components/settings/settingsLayout.tsx
+++ b/apps/web/src/components/settings/settingsLayout.tsx
@@ -172,6 +172,8 @@ export function useSettingsSearchTarget<T extends HTMLElement>(id: string | unde
export function SettingsSearchTarget({
children,
+ // `searchableSetting` names the target for search; as an HTML title it shows a native tooltip.
+ title: _searchTitle,
...targetProps
}: ComponentPropsWithoutRef<"div">) {
const targetRef = useSettingsSearchTarget<HTMLDivElement>(targetProps.id);
@@ -583,10 +585,12 @@ export function SettingsPageContainer({
children,
className,
width = "readable",
+ searchTargetFallbacks,
}: {
children: ReactNode;
className?: string;
width?: WorkspacePageWidth;
+ searchTargetFallbacks?: Readonly<Record<string, string | undefined>>;
}) {
const navigate = useNavigate();
const hash = useLocation({ select: (location) => location.hash });
@@ -606,7 +610,7 @@ export function SettingsPageContainer({
return (
<SettingsSearchTargetProvider
- targetId={targetId}
+ targetId={targetId === null ? null : (searchTargetFallbacks?.[targetId] ?? targetId)}
highlightTarget={highlightTarget}
onTargetHandled={clearTargetHash}
>
diff --git a/apps/web/src/components/settings/settingsSearch.test.ts b/apps/web/src/components/settings/settingsSearch.test.ts
index e5c126e0307..0b26c01f0de 100644
--- a/apps/web/src/components/settings/settingsSearch.test.ts
+++ b/apps/web/src/components/settings/settingsSearch.test.ts
@@ -232,9 +232,12 @@ describe("settings search targets", () => {
expect(isSettingsSearchScopeAvailable(fallback.scope, "environment")).toBe(true);
expect(isSettingsSearchScopeAvailable(fallback.scope, "all")).toBe(true);
expect(isSettingsSearchScopeAvailable(fallback.scope, "project")).toBe(false);
- const streaming = getSettingsSearchTargetScope("response-streaming")!;
- expect(streaming.scope).toBe("project-defaults");
- expect(isSettingsSearchScopeAvailable(streaming.scope, "project")).toBe(true);
+ for (const id of ["response-streaming", "speech-project-dictionary"]) {
+ expect(getSettingsSearchTargetScope(id)?.scope).toBe("project-defaults");
+ }
+ for (const scope of ["all", "environment", "project"] as const) {
+ expect(isSettingsSearchScopeAvailable("project-defaults", scope)).toBe(true);
+ }
for (const id of ["legacy-plan-mode", "legacy-context-window-indicator", "boring-sidebar"]) {
expect(getSettingsSearchTargetScope(id)!.scope).toBeNull();
}
diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts
index 23d32bb35e9..f7156f33bb3 100644
--- a/apps/web/src/components/settings/settingsSearch.ts
+++ b/apps/web/src/components/settings/settingsSearch.ts
@@ -19,6 +19,7 @@ export type SettingsPath =
| "/settings/projects"
| "/settings/general"
| "/settings/diagnostics"
+ | "/settings/voice"
| "/settings/appearance"
| "/settings/chat"
| "/settings/diffs"
@@ -107,6 +108,7 @@ export const SETTINGS_SECTION_LABELS: Readonly<Record<SettingsPath, string>> = {
"/settings/maintenance": "Maintenance",
"/settings/labs": "Labs",
"/settings/about": "About",
+ "/settings/voice": "Voice",
"/settings/archived": "Archive",
};
@@ -123,6 +125,7 @@ export const SETTINGS_NAV_GROUPS: ReadonlyArray<{
"/settings/chat",
"/settings/diffs",
"/settings/keybindings",
+ "/settings/voice",
"/settings/snap-shot",
],
},
@@ -486,6 +489,102 @@ export const SETTINGS_SEARCH_ITEMS = [
scope: "environment",
searchTerms: ["beta alpha simlock callstack simulator emulator ios android pool lease gateway"],
},
+ {
+ id: "voice-transcription-destination",
+ title: "Send microphone audio to",
+ to: "/settings/voice",
+ searchTerms: ["speech voice dictation stt transcription environment machine consent privacy"],
+ },
+ {
+ id: "voice-microphone",
+ title: "Microphone",
+ to: "/settings/voice",
+ searchTerms: ["mic input audio device"],
+ },
+ {
+ id: "dictation-shortcut-mode",
+ title: "Dictation shortcut",
+ to: "/settings/voice",
+ searchTerms: ["speech voice keyboard hotkey hold toggle auto"],
+ },
+ {
+ id: "speech-language",
+ title: "Transcription language",
+ to: "/settings/voice",
+ searchTerms: ["speech voice translate english auto detect"],
+ },
+ {
+ id: "speech-dictionary",
+ title: "Dictionary",
+ to: "/settings/voice",
+ searchTerms: ["speech voice transcription vocabulary names glossary alias"],
+ },
+ {
+ id: "speech-project-dictionary",
+ title: "Project dictionary",
+ to: "/settings/voice",
+ scope: "project-defaults",
+ searchTerms: [
+ "speech voice transcription vocabulary shared dictionary team environment glossary",
+ ],
+ },
+ {
+ id: "speech-post-processing-model",
+ title: "Cleanup model",
+ to: "/settings/voice",
+ scope: "environment",
+ searchTerms: ["speech voice transcript cleanup provider model post-processing"],
+ },
+ {
+ id: "speech-correction-word",
+ title: "Correction cue",
+ to: "/settings/voice",
+ searchTerms: ["speech voice cleanup sorry scratch that fix mistake"],
+ },
+ {
+ id: "speech-silence-filtering",
+ title: "Silence filtering",
+ to: "/settings/voice",
+ searchTerms: ["voice speech vad silero pauses noise"],
+ },
+ {
+ id: "remove-filler-words",
+ title: "Remove filler words",
+ to: "/settings/voice",
+ searchTerms: ["voice speech transcription hesitation um uh cleanup"],
+ },
+ {
+ id: "speech-post-processing",
+ title: "Clean up transcripts with a provider",
+ to: "/settings/voice",
+ searchTerms: ["speech transcription polish clean provider prompt post-processing"],
+ },
+ {
+ id: "speech-models",
+ title: "Transcription models",
+ to: "/settings/voice",
+ searchTerms: [
+ "speech voice stt download whisper parakeet canary moonshine accuracy speed benchmark languages",
+ ],
+ },
+ {
+ id: "speech-hardware",
+ title: "Transcription hardware and model unloading",
+ to: "/settings/voice",
+ searchTerms: ["speech voice stt advanced gpu cpu acceleration memory unload idle"],
+ },
+ {
+ id: "microphone-test",
+ title: "Test microphone",
+ to: "/settings/voice",
+ searchTerms: ["test mic audio playback loopback record input level"],
+ },
+ {
+ id: "transcription-test",
+ title: "Test model",
+ to: "/settings/voice",
+ searchTerms: ["test voice speech model transcription recognition"],
+ },
{
id: "auto-settle-inactive-threads",
title: "Auto-settle inactive threads",
@@ -1202,6 +1301,7 @@ const SETTINGS_CATEGORY_SCOPES: Readonly<Record<SettingsPath, SettingsSearchScop
"/settings/diagnostics": "environment",
"/settings/projects": "project",
"/settings/general": null,
+ "/settings/voice": null,
"/settings/appearance": null,
"/settings/chat": null,
"/settings/diffs": null,
diff --git a/apps/web/src/components/ui/button.tsx b/apps/web/src/components/ui/button.tsx
index 8aa747ad362..5b36a5ae5eb 100644
--- a/apps/web/src/components/ui/button.tsx
+++ b/apps/web/src/components/ui/button.tsx
@@ -23,6 +23,7 @@ const buttonVariants = cva(
"size-5 rounded-sm p-0 before:rounded-[calc(var(--radius-sm)-1px)] [&_svg:not([class*='size-'])]:size-3",
"icon-tiny": "size-4 p-0 [&_svg:not([class*='size-'])]:size-3",
"icon-sm": "size-8 sm:size-7",
+ "icon-sm-pill": "size-8 rounded-full before:rounded-full sm:size-7",
"icon-xl":
"size-11 sm:size-10 [&_svg:not([class*='size-'])]:size-5 sm:[&_svg:not([class*='size-'])]:size-4.5",
"icon-xs":
@@ -33,6 +34,7 @@ const buttonVariants = cva(
multiline:
"min-h-9 px-[calc(--spacing(3)-1px)] py-[calc(--spacing(1.5)-1px)] whitespace-normal sm:min-h-8",
sm: "h-8 gap-1.5 px-[calc(--spacing(2.5)-1px)] sm:h-7",
+ "sm-pill": "h-8 gap-1.5 rounded-full px-2.5 sm:h-7",
"sm-multiline":
"min-h-8 gap-1.5 px-[calc(--spacing(2.5)-1px)] py-[calc(--spacing(1)-1px)] whitespace-normal sm:min-h-7",
xl: "h-11 px-[calc(--spacing(4)-1px)] text-lg sm:h-10 sm:text-base [&_svg:not([class*='size-'])]:size-5 sm:[&_svg:not([class*='size-'])]:size-4.5",
diff --git a/apps/web/src/components/ui/textarea.tsx b/apps/web/src/components/ui/textarea.tsx
index 681d1f043f3..116d0d663fb 100644
--- a/apps/web/src/components/ui/textarea.tsx
+++ b/apps/web/src/components/ui/textarea.tsx
@@ -9,7 +9,13 @@ type TextareaProps = React.ComponentProps<"textarea"> & {
unstyled?: boolean;
};
-function Textarea({ className, size = "default", unstyled = false, ...props }: TextareaProps) {
+function Textarea({
+ className,
+ size = "default",
+ unstyled = false,
+ rows,
+ ...props
+}: TextareaProps) {
return (
<span
className={
@@ -26,12 +32,23 @@ function Textarea({ className, size = "default", unstyled = false, ...props }: T
render={(defaultProps) => (
<textarea
className={cn(
- "field-sizing-content min-h-17.5 max-h-64 w-full rounded-[inherit] px-[calc(--spacing(3)-1px)] py-[calc(--spacing(1.5)-1px)] outline-none wrap-anywhere max-sm:min-h-20.5",
- size === "sm" &&
- "min-h-16.5 px-[calc(--spacing(2.5)-1px)] py-[calc(--spacing(1)-1px)] max-sm:min-h-19.5",
- size === "lg" && "min-h-18.5 py-[calc(--spacing(2)-1px)] max-sm:min-h-21.5",
+ "field-sizing-content max-h-64 w-full rounded-[inherit] px-[calc(--spacing(3)-1px)] py-[calc(--spacing(1.5)-1px)] outline-none wrap-anywhere",
+ rows === undefined && "min-h-17.5 max-sm:min-h-20.5",
+ size === "sm" && "px-[calc(--spacing(2.5)-1px)] py-[calc(--spacing(1)-1px)]",
+ size === "sm" && rows === undefined && "min-h-16.5 max-sm:min-h-19.5",
+ size === "lg" && "py-[calc(--spacing(2)-1px)]",
+ size === "lg" && rows === undefined && "min-h-18.5 max-sm:min-h-21.5",
)}
+ // field-sizing: content ignores `rows`, so an explicit row count sets the minimum height.
+ style={
+ rows === undefined
+ ? undefined
+ : {
+ minHeight: `calc(${rows}lh + 2 * (var(--spacing) * ${size === "sm" ? 1 : size === "lg" ? 2 : 1.5} - 1px))`,
+ }
+ }
data-slot="textarea"
+ rows={rows}
{...mergeProps(defaultProps, props)}
/>
)}
diff --git a/apps/web/src/hooks/useLocalStorage.ts b/apps/web/src/hooks/useLocalStorage.ts
index 5c172287131..6d3834286c7 100644
--- a/apps/web/src/hooks/useLocalStorage.ts
+++ b/apps/web/src/hooks/useLocalStorage.ts
@@ -134,7 +134,7 @@ export function readLocalStorageValue<T, E>(
/** Fires on writes from this tab and from other tabs. */
export function subscribeLocalStorageKey(key: string, onChange: () => void): () => void {
const handleStorageChange = (event: StorageEvent) => {
- if (event.key === key) {
+ if (event.key === key || event.key === null) {
onChange();
}
};
diff --git a/apps/web/src/keybindings.test.ts b/apps/web/src/keybindings.test.ts
index 1f0a096fcca..8708bbd297a 100644
--- a/apps/web/src/keybindings.test.ts
+++ b/apps/web/src/keybindings.test.ts
@@ -767,6 +767,23 @@ describe("isRichTextBoldShortcut", () => {
});
describe("composer and pull request shortcuts", () => {
+ it("uses the dictation chord outside the terminal and preserves terminal split", () => {
+ const chord = event({ key: "d", ctrlKey: true, shiftKey: true });
+ assert.strictEqual(
+ resolveShortcutCommand(chord, DEFAULT_RESOLVED_KEYBINDINGS, {
+ platform: "Linux",
+ context: { terminalFocus: false },
+ }),
+ "composer.dictation",
+ );
+ assert.strictEqual(
+ resolveShortcutCommand(chord, DEFAULT_RESOLVED_KEYBINDINGS, {
+ platform: "Linux",
+ context: { terminalFocus: true },
+ }),
+ "terminal.splitVertical",
+ );
+ });
it("fills missing number shortcuts without replacing the saved URL binding", () => {
const olderServerBindings = DEFAULT_RESOLVED_KEYBINDINGS.filter(
(binding) =>
diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts
index 33843126b2a..cbcebdd6790 100644
--- a/apps/web/src/routeTree.gen.ts
+++ b/apps/web/src/routeTree.gen.ts
@@ -23,6 +23,7 @@ import { Route as ChatPullRequestsRouteImport } from './routes/_chat.pull-reques
import { Route as ChatNewRouteImport } from './routes/_chat.new'
import { Route as ThreadsEnvironmentIdThreadIdRouteImport } from './routes/threads.$environmentId.$threadId'
import { Route as ChatSettingsWorktreesRouteImport } from './routes/_chat.settings.worktrees'
+import { Route as ChatSettingsVoiceRouteImport } from './routes/_chat.settings.voice'
import { Route as ChatSettingsThreadsRouteImport } from './routes/_chat.settings.threads'
import { Route as ChatSettingsStorageRouteImport } from './routes/_chat.settings.storage'
import { Route as ChatSettingsSourceControlRouteImport } from './routes/_chat.settings.source-control'
@@ -121,6 +122,11 @@ const ChatSettingsWorktreesRoute = ChatSettingsWorktreesRouteImport.update({
path: '/worktrees',
getParentRoute: () => ChatSettingsRoute,
} as any)
+const ChatSettingsVoiceRoute = ChatSettingsVoiceRouteImport.update({
+ id: '/voice',
+ path: '/voice',
+ getParentRoute: () => ChatSettingsRoute,
+} as any)
const ChatSettingsThreadsRoute = ChatSettingsThreadsRouteImport.update({
id: '/threads',
path: '/threads',
@@ -301,6 +307,7 @@ export interface FileRoutesByFullPath {
'/settings/source-control': typeof ChatSettingsSourceControlRoute
'/settings/storage': typeof ChatSettingsStorageRoute
'/settings/threads': typeof ChatSettingsThreadsRoute
+ '/settings/voice': typeof ChatSettingsVoiceRoute
'/settings/worktrees': typeof ChatSettingsWorktreesRoute
'/threads/$environmentId/$threadId': typeof ThreadsEnvironmentIdThreadIdRoute
}
@@ -343,6 +350,7 @@ export interface FileRoutesByTo {
'/settings/source-control': typeof ChatSettingsSourceControlRoute
'/settings/storage': typeof ChatSettingsStorageRoute
'/settings/threads': typeof ChatSettingsThreadsRoute
+ '/settings/voice': typeof ChatSettingsVoiceRoute
'/settings/worktrees': typeof ChatSettingsWorktreesRoute
'/threads/$environmentId/$threadId': typeof ThreadsEnvironmentIdThreadIdRoute
}
@@ -387,6 +395,7 @@ export interface FileRoutesById {
'/_chat/settings/source-control': typeof ChatSettingsSourceControlRoute
'/_chat/settings/storage': typeof ChatSettingsStorageRoute
'/_chat/settings/threads': typeof ChatSettingsThreadsRoute
+ '/_chat/settings/voice': typeof ChatSettingsVoiceRoute
'/_chat/settings/worktrees': typeof ChatSettingsWorktreesRoute
'/threads/$environmentId/$threadId': typeof ThreadsEnvironmentIdThreadIdRoute
}
@@ -431,6 +440,7 @@ export interface FileRouteTypes {
| '/settings/source-control'
| '/settings/storage'
| '/settings/threads'
+ | '/settings/voice'
| '/settings/worktrees'
| '/threads/$environmentId/$threadId'
fileRoutesByTo: FileRoutesByTo
@@ -473,6 +483,7 @@ export interface FileRouteTypes {
| '/settings/source-control'
| '/settings/storage'
| '/settings/threads'
+ | '/settings/voice'
| '/settings/worktrees'
| '/threads/$environmentId/$threadId'
id:
@@ -516,6 +527,7 @@ export interface FileRouteTypes {
| '/_chat/settings/source-control'
| '/_chat/settings/storage'
| '/_chat/settings/threads'
+ | '/_chat/settings/voice'
| '/_chat/settings/worktrees'
| '/threads/$environmentId/$threadId'
fileRoutesById: FileRoutesById
@@ -631,6 +643,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof ChatSettingsWorktreesRouteImport
parentRoute: typeof ChatSettingsRoute
}
+ '/_chat/settings/voice': {
+ id: '/_chat/settings/voice'
+ path: '/voice'
+ fullPath: '/settings/voice'
+ preLoaderRoute: typeof ChatSettingsVoiceRouteImport
+ parentRoute: typeof ChatSettingsRoute
+ }
'/_chat/settings/threads': {
id: '/_chat/settings/threads'
path: '/threads'
@@ -849,6 +868,7 @@ interface ChatSettingsRouteChildren {
ChatSettingsSourceControlRoute: typeof ChatSettingsSourceControlRoute
ChatSettingsStorageRoute: typeof ChatSettingsStorageRoute
ChatSettingsThreadsRoute: typeof ChatSettingsThreadsRoute
+ ChatSettingsVoiceRoute: typeof ChatSettingsVoiceRoute
ChatSettingsWorktreesRoute: typeof ChatSettingsWorktreesRoute
}
@@ -878,6 +898,7 @@ const ChatSettingsRouteChildren: ChatSettingsRouteChildren = {
ChatSettingsSourceControlRoute: ChatSettingsSourceControlRoute,
ChatSettingsStorageRoute: ChatSettingsStorageRoute,
ChatSettingsThreadsRoute: ChatSettingsThreadsRoute,
+ ChatSettingsVoiceRoute: ChatSettingsVoiceRoute,
ChatSettingsWorktreesRoute: ChatSettingsWorktreesRoute,
}
diff --git a/apps/web/src/routes/_chat.settings.voice.tsx b/apps/web/src/routes/_chat.settings.voice.tsx
new file mode 100644
index 00000000000..d577ff4b616
--- /dev/null
+++ b/apps/web/src/routes/_chat.settings.voice.tsx
@@ -0,0 +1,7 @@
+import { createFileRoute } from "@tanstack/react-router";
+
+import { VoiceSettingsPanel } from "../components/settings/VoiceSettingsPanel";
+
+export const Route = createFileRoute("/_chat/settings/voice")({
+ component: VoiceSettingsPanel,
+});
diff --git a/apps/web/src/speech/browserMicrophones.ts b/apps/web/src/speech/browserMicrophones.ts
new file mode 100644
index 00000000000..f690c4fcd5f
--- /dev/null
+++ b/apps/web/src/speech/browserMicrophones.ts
@@ -0,0 +1,11 @@
+export type BrowserMicrophone = Pick<MediaDeviceInfo, "deviceId" | "label">;
+
+export async function listBrowserMicrophones(): Promise<BrowserMicrophone[]> {
+ const devices = await navigator.mediaDevices.enumerateDevices();
+ return devices
+ .filter((device) => device.kind === "audioinput")
+ .map(({ deviceId, label }, index) => ({
+ deviceId,
+ label: label || `Microphone ${index + 1}`,
+ }));
+}
diff --git a/apps/web/src/speech/browserPcmCapture.browser.test.ts b/apps/web/src/speech/browserPcmCapture.browser.test.ts
new file mode 100644
index 00000000000..892f6d64069
--- /dev/null
+++ b/apps/web/src/speech/browserPcmCapture.browser.test.ts
@@ -0,0 +1,28 @@
+import { expect, it, vi } from "vite-plus/test";
+import { createBrowserPcmCapture } from "./browserPcmCapture";
+
+it("cancels an unanswered permission prompt and stops a late microphone stream", async () => {
+ let grantPermission!: (media: MediaStream) => void;
+ const permission = new Promise<MediaStream>((resolve) => {
+ grantPermission = resolve;
+ });
+ using microphone = vi.spyOn(navigator.mediaDevices, "getUserMedia").mockReturnValue(permission);
+ const capture = createBrowserPcmCapture(
+ { getMicrophoneId: () => "", onLevel: vi.fn(), onDurationLimit: vi.fn(), onError: vi.fn() },
+ async () => ({ locale: "en", transcribe: async () => "" }),
+ );
+ const abort = new AbortController();
+ await capture.transcriber.prepare({ signal: abort.signal });
+ const rejected = vi.fn();
+ const preparing = capture.recorder.prepareToRecordAsync(abort.signal).catch(rejected);
+ expect(microphone).toHaveBeenCalledOnce();
+ abort.abort();
+ const stop = vi.fn();
+ try {
+ await vi.waitFor(() => expect(rejected).toHaveBeenCalledWith(abort.signal.reason));
+ } finally {
+ grantPermission({ getTracks: () => [{ stop }] } as unknown as MediaStream);
+ await preparing;
+ }
+ expect(stop).toHaveBeenCalledOnce();
+});
diff --git a/apps/web/src/speech/browserPcmCapture.ts b/apps/web/src/speech/browserPcmCapture.ts
new file mode 100644
index 00000000000..f38d616d954
--- /dev/null
+++ b/apps/web/src/speech/browserPcmCapture.ts
@@ -0,0 +1,180 @@
+import type {
+ PreparedVoiceTranscription,
+ VoiceRecorder,
+ VoiceStreamingSession,
+ VoiceTranscriptionOptions,
+} from "@t3tools/client-runtime/voice-input";
+import workletUrl from "./pcmWorklet.ts?worker&url";
+
+type PreparedBrowserTranscription = PreparedVoiceTranscription & {
+ readonly streaming?: VoiceStreamingSession & { readonly feed: (pcm: Float32Array) => void };
+};
+
+export function createBrowserPcmCapture(
+ input: {
+ readonly getMicrophoneId: () => string;
+ readonly onLevel: (level: number) => void;
+ readonly onDurationLimit: () => void;
+ readonly onError: (message: string) => void;
+ },
+ prepare: (options: VoiceTranscriptionOptions) => Promise<PreparedBrowserTranscription>,
+) {
+ let stream: MediaStream | undefined;
+ let recordingUri: string | null = null;
+ let chunks: Float32Array<ArrayBuffer>[] = [];
+ let audioContext: AudioContext | undefined;
+ let worklet: AudioWorkletNode | undefined;
+ let durationTimer: ReturnType<typeof setTimeout> | undefined;
+ let signal: AbortSignal | undefined;
+ let feed: ((pcm: Float32Array) => void) | undefined;
+ let stopped:
+ | { readonly resolve: () => void; readonly reject: (error: Error) => void }
+ | undefined;
+
+ const cleanupCapture = () => {
+ if (durationTimer) clearTimeout(durationTimer);
+ durationTimer = undefined;
+ worklet?.disconnect();
+ worklet?.port.close();
+ worklet = undefined;
+ stream?.getTracks().forEach((track) => track.stop());
+ stream = undefined;
+ void audioContext?.close();
+ audioContext = undefined;
+ stopped?.resolve();
+ stopped = undefined;
+ chunks = [];
+ feed = undefined;
+ input.onLevel(0);
+ };
+
+ const recorder: VoiceRecorder = {
+ get uri() {
+ return recordingUri;
+ },
+ prepareToRecordAsync: async () => {
+ const captureSignal = signal;
+ if (!captureSignal) throw new Error("Voice transcription is not prepared.");
+ captureSignal.throwIfAborted();
+ if (recordingUri) URL.revokeObjectURL(recordingUri);
+ recordingUri = null;
+ chunks = [];
+ const microphoneId = input.getMicrophoneId();
+ const captured = await new Promise<MediaStream>((resolve, reject) => {
+ const onAbort = () => reject(captureSignal.reason);
+ const permission = navigator.mediaDevices.getUserMedia({
+ audio: microphoneId ? { deviceId: { exact: microphoneId } } : true,
+ });
+ captureSignal.addEventListener("abort", onAbort, { once: true });
+ void permission.then(
+ (media) => {
+ captureSignal.removeEventListener("abort", onAbort);
+ if (captureSignal.aborted) media.getTracks().forEach((track) => track.stop());
+ else resolve(media);
+ },
+ (error: unknown) => {
+ captureSignal.removeEventListener("abort", onAbort);
+ reject(error);
+ },
+ );
+ });
+ if (captureSignal.aborted) {
+ captured.getTracks().forEach((track) => track.stop());
+ captureSignal.throwIfAborted();
+ }
+ stream = captured;
+ try {
+ const context = new AudioContext();
+ audioContext = context;
+ await context.audioWorklet.addModule(workletUrl);
+ captureSignal.throwIfAborted();
+ const node = new AudioWorkletNode(context, "t3-pcm-capture", { channelCount: 1 });
+ worklet = node;
+ node.port.onmessage = ({
+ data,
+ }: MessageEvent<Float32Array<ArrayBuffer> | number | "stopped" | "limit">) => {
+ if (captureSignal.aborted) return;
+ if (data === "stopped") {
+ stopped?.resolve();
+ return;
+ }
+ if (data === "limit") {
+ input.onDurationLimit();
+ return;
+ }
+ if (typeof data === "number") {
+ input.onLevel(data);
+ return;
+ }
+ if (feed) feed(data);
+ else chunks.push(data);
+ };
+ node.onprocessorerror = () => {
+ stopped?.reject(new Error("Microphone processing failed."));
+ input.onError("Microphone processing failed.");
+ };
+ for (const track of captured.getTracks())
+ track.addEventListener("ended", () => input.onError("The microphone disconnected."), {
+ once: true,
+ });
+ context.createMediaStreamSource(captured).connect(node);
+ // The processor emits silence, keeping the graph active without microphone playback.
+ node.connect(context.destination);
+ await context.resume();
+ captureSignal.throwIfAborted();
+ } catch (error) {
+ cleanupCapture();
+ throw error;
+ }
+ },
+ record: ({ forDuration }) => {
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
+ worklet?.port.postMessage("start");
+ durationTimer = setTimeout(input.onDurationLimit, forDuration * 1_000);
+ },
+ stop: async () => {
+ if (!worklet) return;
+ if (signal?.aborted) {
+ cleanupCapture();
+ return;
+ }
+ const pending = new Promise<void>((resolve, reject) => {
+ stopped = { resolve, reject };
+ });
+ const timeout = setTimeout(
+ () => stopped?.reject(new Error("Microphone capture did not stop.")),
+ 5_000,
+ );
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
+ worklet.port.postMessage("stop");
+ try {
+ await pending;
+ if (!feed && chunks.length) {
+ const blob = new Blob(chunks, { type: "application/octet-stream" });
+ recordingUri = URL.createObjectURL(blob);
+ }
+ } finally {
+ clearTimeout(timeout);
+ cleanupCapture();
+ }
+ },
+ };
+
+ return {
+ recorder,
+ cancelRecording: cleanupCapture,
+ deleteRecording: (uri: string) => {
+ URL.revokeObjectURL(uri);
+ if (recordingUri === uri) recordingUri = null;
+ },
+ transcriber: {
+ prepare: async (options: VoiceTranscriptionOptions) => {
+ const prepared = await prepare(options);
+ options.signal.throwIfAborted();
+ signal = options.signal;
+ feed = prepared.streaming?.feed;
+ return prepared;
+ },
+ },
+ };
+}
diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts
new file mode 100644
index 00000000000..1d04f96bba0
--- /dev/null
+++ b/apps/web/src/speech/browserVoiceInput.ts
@@ -0,0 +1,89 @@
+import { effectiveSpeechLanguage } from "@t3tools/shared/speech";
+import {
+ getEnvironmentSpeechStatus,
+ getEnvironmentSpeechStreamUrl,
+ openSpeechStream,
+ throwIfVoiceTranscriptionAborted,
+ transcribeEnvironmentPcm,
+ VoiceTranscriptionError,
+ type VoiceTranscriptionOptions,
+} from "@t3tools/client-runtime/voice-input";
+import type { PreparedConnection } from "@t3tools/client-runtime/connection";
+import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts";
+
+import { runtime } from "../lib/runtime";
+import { createBrowserPcmCapture } from "./browserPcmCapture";
+
+export function createBrowserVoiceInputPlatform(input: {
+ readonly prepared: PreparedConnection;
+ readonly hasAudioConsent: () => boolean;
+ readonly getTranscriptionOptions: () => SpeechTranscriptionOptions;
+ readonly getMicrophoneId: () => string;
+ readonly onLevel: (level: number) => void;
+ readonly onDurationLimit: () => void;
+ readonly onText: (text: SpeechStreamText) => void;
+ readonly onError: (message: string) => void;
+}) {
+ return createBrowserPcmCapture(input, async (options) => {
+ const recordingOptions = input.getTranscriptionOptions();
+ const status = await runtime.runPromise(getEnvironmentSpeechStatus(input.prepared), options);
+ throwIfVoiceTranscriptionAborted(options.signal);
+ if (!status.supported)
+ throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
+ const language = status.languages
+ ? effectiveSpeechLanguage(
+ {
+ languages: status.languages,
+ supportsLanguageDetection: status.supportsLanguageDetection ?? false,
+ },
+ recordingOptions.speechLanguage,
+ )
+ : "auto";
+ const locale =
+ !status.supportsStreaming &&
+ status.supportsTranslation &&
+ recordingOptions.speechTranslateToEnglish
+ ? "en"
+ : language;
+ const transcribeRecording = async (
+ uri: string,
+ { signal: transcriptionSignal }: VoiceTranscriptionOptions,
+ ) => {
+ try {
+ const response = await fetch(uri, { signal: transcriptionSignal });
+ const pcm = new Uint8Array(await response.arrayBuffer());
+ const result = await runtime.runPromise(
+ transcribeEnvironmentPcm(input.prepared, pcm, recordingOptions, input.hasAudioConsent),
+ {
+ signal: transcriptionSignal,
+ },
+ );
+ throwIfVoiceTranscriptionAborted(transcriptionSignal);
+ return result.text;
+ } catch (cause) {
+ throwIfVoiceTranscriptionAborted(transcriptionSignal);
+ throw new VoiceTranscriptionError({
+ code: "transcription-failed",
+ message: "Voice transcription on this environment failed.",
+ cause,
+ });
+ }
+ };
+ if (!status.supportsStreaming && !status.supportsBufferedStreaming) {
+ return { locale, transcribe: transcribeRecording };
+ }
+ const url = await runtime.runPromise(getEnvironmentSpeechStreamUrl(input.prepared), options);
+ return {
+ locale,
+ transcribe: transcribeRecording,
+ streaming: openSpeechStream({
+ url,
+ canSend: input.hasAudioConsent,
+ options: recordingOptions,
+ signal: options.signal,
+ onText: input.onText,
+ onError: (error) => input.onError(error.message),
+ }),
+ };
+ });
+}
diff --git a/apps/web/src/speech/pcmResampler.ts b/apps/web/src/speech/pcmResampler.ts
new file mode 100644
index 00000000000..3e63c9011d0
--- /dev/null
+++ b/apps/web/src/speech/pcmResampler.ts
@@ -0,0 +1,54 @@
+const TARGET_RATE = 16_000;
+
+/** Low-pass before downsampling, preserving filter and sample phase across capture blocks. */
+export class PcmResampler {
+ private readonly coefficients: Float64Array;
+ private readonly history: Float32Array;
+ private position = 0;
+ private phase = 0;
+ private readonly ratio: number;
+
+ constructor(sampleRate: number) {
+ if (sampleRate < TARGET_RATE)
+ throw new Error("Microphone sample rate must be at least 16 kHz.");
+ this.ratio = sampleRate / TARGET_RATE;
+ this.coefficients = new Float64Array(sampleRate === TARGET_RATE ? 1 : 63);
+ this.history = new Float32Array(this.coefficients.length);
+ const middle = (this.coefficients.length - 1) / 2;
+ const cutoff = 7_200 / sampleRate;
+ let sum = 0;
+ for (let i = 0; i < this.coefficients.length; i++) {
+ const x = i - middle;
+ const sinc = x === 0 ? 2 * cutoff : Math.sin(2 * Math.PI * cutoff * x) / (Math.PI * x);
+ const window = middle === 0 ? 1 : 0.54 - 0.46 * Math.cos((Math.PI * i) / middle);
+ this.coefficients[i] = sinc * window;
+ sum += this.coefficients[i]!;
+ }
+ for (let i = 0; i < this.coefficients.length; i++) this.coefficients[i]! /= sum;
+ }
+
+ push(samples: Float32Array): Float32Array {
+ const output: number[] = [];
+ for (const sample of samples) {
+ this.history[this.position] = sample;
+ this.phase += 1;
+ if (this.phase >= this.ratio) {
+ this.phase -= this.ratio;
+ let value = 0;
+ for (let tap = 0; tap < this.coefficients.length; tap++) {
+ const index = (this.position - tap + this.history.length) % this.history.length;
+ value += this.history[index]! * this.coefficients[tap]!;
+ }
+ output.push(Math.max(-1, Math.min(1, value)));
+ }
+ this.position = (this.position + 1) % this.history.length;
+ }
+ return Float32Array.from(output);
+ }
+
+ flush(): Float32Array {
+ return this.coefficients.length === 1
+ ? new Float32Array()
+ : this.push(new Float32Array(this.history.length - 1));
+ }
+}
diff --git a/apps/web/src/speech/pcmWorklet.test.ts b/apps/web/src/speech/pcmWorklet.test.ts
new file mode 100644
index 00000000000..6cd35c2de5e
--- /dev/null
+++ b/apps/web/src/speech/pcmWorklet.test.ts
@@ -0,0 +1,64 @@
+import { expect, it, vi } from "vite-plus/test";
+
+it("meters microphone-only capture like transcription without emitting PCM", async () => {
+ using globals = new DisposableStack();
+ const setGlobal = (name: string, value: unknown) => {
+ const original = Object.getOwnPropertyDescriptor(globalThis, name);
+ globals.defer(() => {
+ if (original) Object.defineProperty(globalThis, name, original);
+ else Reflect.deleteProperty(globalThis, name);
+ });
+ Object.defineProperty(globalThis, name, { value, configurable: true, writable: true });
+ };
+ class Worklet {
+ readonly port = {
+ onmessage: null as ((event: { data: string }) => void) | null,
+ postMessage: vi.fn<(data: unknown) => void>(),
+ };
+ process(_inputs: Float32Array[][]): boolean {
+ return true;
+ }
+ }
+ let Processor!: new (options?: { processorOptions: { meterOnly: boolean } }) => Worklet;
+ setGlobal("sampleRate", 48_000);
+ setGlobal("AudioWorkletProcessor", Worklet);
+ setGlobal("registerProcessor", (_name: string, processor: typeof Processor) => {
+ Processor = processor;
+ });
+ await import("./pcmWorklet");
+ const transcription = new Processor();
+ const microphone = new Processor({ processorOptions: { meterOnly: true } });
+ transcription.port.onmessage?.({ data: "start" });
+ microphone.port.onmessage?.({ data: "start" });
+ const feed = (samples: Float32Array) => {
+ transcription.process([[samples]]);
+ microphone.process([[samples]]);
+ };
+ const levels = (processor: Worklet) =>
+ processor.port.postMessage.mock.calls
+ .map(([data]) => data)
+ .filter((data) => typeof data === "number");
+
+ feed(new Float32Array(2_399).fill(0.1));
+ expect(levels(microphone)).toHaveLength(0);
+ feed(new Float32Array(1).fill(0.1));
+ expect(levels(microphone)).toHaveLength(1);
+ feed(new Float32Array(48_000).fill(0.25));
+ feed(new Float32Array(2_400));
+
+ const measured = levels(microphone);
+ expect(measured).toEqual(levels(transcription));
+ expect(measured).toHaveLength(22);
+ expect(measured[0]).toBeGreaterThan(0);
+ expect(measured[0]).toBeLessThan(0.4);
+ expect(measured.at(-1)).toBeGreaterThan(0);
+ expect(measured.at(-1)).toBeLessThan(measured.at(-2)!);
+ expect(
+ transcription.port.postMessage.mock.calls.some(([data]) => data instanceof Float32Array),
+ ).toBe(true);
+ expect(microphone.port.postMessage.mock.calls.every(([data]) => typeof data === "number")).toBe(
+ true,
+ );
+ microphone.port.onmessage?.({ data: "stop" });
+ expect(microphone.port.postMessage).toHaveBeenLastCalledWith("stopped");
+});
diff --git a/apps/web/src/speech/pcmWorklet.ts b/apps/web/src/speech/pcmWorklet.ts
new file mode 100644
index 00000000000..ff6962ce530
--- /dev/null
+++ b/apps/web/src/speech/pcmWorklet.ts
@@ -0,0 +1,84 @@
+import { PcmResampler } from "./pcmResampler";
+
+declare const sampleRate: number;
+declare class AudioWorkletProcessor {
+ readonly port: MessagePort;
+}
+declare function registerProcessor(name: string, processor: typeof AudioWorkletProcessor): void;
+
+class PcmCaptureProcessor extends AudioWorkletProcessor {
+ private readonly resampler = new PcmResampler(sampleRate);
+ private chunk = new Float32Array(8_000);
+ private length = 0;
+ private recording = false;
+ private flushed = false;
+ private samples = 0;
+ private levelSamples = 0;
+ private levelEnergy = 0;
+ private level = 0;
+ private readonly meterOnly: boolean;
+
+ constructor(options?: { readonly processorOptions?: { readonly meterOnly?: boolean } }) {
+ super();
+ this.meterOnly = options?.processorOptions?.meterOnly === true;
+ this.port.onmessage = ({ data }) => {
+ if (data === "start") this.recording = true;
+ if (data === "stop") {
+ this.recording = false;
+ if (!this.meterOnly && !this.flushed) this.append(this.resampler.flush());
+ this.flushed = true;
+ if (this.length) this.publish();
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
+ this.port.postMessage("stopped");
+ }
+ };
+ }
+
+ private publish() {
+ const pcm = this.chunk.slice(0, this.length);
+ this.port.postMessage(pcm, [pcm.buffer]);
+ this.length = 0;
+ }
+
+ private append(pcm: Float32Array) {
+ for (const sample of pcm) {
+ if (this.samples === 16_000 * 300) {
+ this.recording = false;
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
+ this.port.postMessage("limit");
+ break;
+ }
+ this.chunk[this.length++] = sample;
+ this.samples++;
+ if (this.length === this.chunk.length) this.publish();
+ }
+ }
+
+ process(inputs: Float32Array[][]): boolean {
+ if (!this.recording) return true;
+ const channels = inputs[0];
+ if (!channels?.[0]) return true;
+ const mono = new Float32Array(channels[0].length);
+ for (const channel of channels)
+ for (let i = 0; i < mono.length; i++) mono[i]! += channel[i]! / channels.length;
+ // Metering must not wait for the half-second transcription chunks.
+ for (const sample of mono) {
+ this.levelEnergy += sample * sample;
+ this.levelSamples++;
+ if (this.levelSamples >= sampleRate / 20) {
+ const target = Math.min(1, Math.sqrt(this.levelEnergy / this.levelSamples) * 4);
+ // Soften brief peaks without delaying meter updates or transcription.
+ const timeConstant = target > this.level ? 0.075 : 0.2;
+ const decay = Math.exp(-this.levelSamples / sampleRate / timeConstant);
+ this.level = target + (this.level - target) * decay;
+ this.port.postMessage(this.level, []);
+ this.levelEnergy = 0;
+ this.levelSamples = 0;
+ }
+ }
+ if (!this.meterOnly) this.append(this.resampler.push(mono));
+ return true;
+ }
+}
+
+registerProcessor("t3-pcm-capture", PcmCaptureProcessor);
diff --git a/apps/web/src/speech/useDictationShortcut.browser.test.tsx b/apps/web/src/speech/useDictationShortcut.browser.test.tsx
new file mode 100644
index 00000000000..8cd17f4f368
--- /dev/null
+++ b/apps/web/src/speech/useDictationShortcut.browser.test.tsx
@@ -0,0 +1,82 @@
+import { DEFAULT_RESOLVED_KEYBINDINGS } from "@t3tools/shared/keybindings";
+import { expect, it, vi } from "vite-plus/test";
+import { render } from "../../test/render";
+import { useDictationShortcut } from "./useDictationShortcut";
+
+vi.mock("../hooks/useSettings", () => ({ useClientSettings: () => "hold" }));
+
+function Probe({ speech }: { speech: Parameters<typeof useDictationShortcut>[0]["speech"] }) {
+ useDictationShortcut({
+ keybindings: DEFAULT_RESOLVED_KEYBINDINGS,
+ speech,
+ disabled: false,
+ terminalOpen: false,
+ modelPickerOpen: false,
+ });
+ return null;
+}
+
+const press = (repeat = false) =>
+ window.dispatchEvent(
+ new KeyboardEvent("keydown", {
+ key: "D",
+ code: "KeyD",
+ ctrlKey: !/Mac/i.test(navigator.platform),
+ metaKey: /Mac/i.test(navigator.platform),
+ shiftKey: true,
+ cancelable: true,
+ repeat,
+ }),
+ );
+
+function actions(available: boolean) {
+ return {
+ available,
+ state: { phase: "idle" as const, error: null, errorAction: null },
+ start: vi.fn(async () => {}),
+ stop: vi.fn(async () => {}),
+ cancel: vi.fn(),
+ };
+}
+
+it("leaves the browser shortcut alone when dictation is off", async () => {
+ const speech = actions(false);
+ await using _screen = await render(<Probe speech={speech} />);
+ expect(press()).toBe(true);
+ expect(speech.start).not.toHaveBeenCalled();
+});
+
+it("stops on a new press when the OS omitted keyup, but ignores autorepeat", async () => {
+ const speech = actions(true);
+ await using screen = await render(<Probe speech={speech} />);
+ expect(press()).toBe(false);
+ expect(speech.start).toHaveBeenCalledOnce();
+ await screen.rerender(
+ <Probe speech={{ ...speech, state: { ...speech.state, phase: "recording" } }} />,
+ );
+ press(true);
+ expect(speech.stop).not.toHaveBeenCalled();
+ press();
+ expect(speech.stop).toHaveBeenCalledOnce();
+});
+
+it("finishes a released hold after startup even if the chord was pressed again while preparing", async () => {
+ const speech = actions(true);
+ let finishStart!: () => void;
+ const startup = new Promise<void>((resolve) => {
+ finishStart = resolve;
+ });
+ speech.start.mockReturnValue(startup);
+ await using screen = await render(<Probe speech={speech} />);
+ press();
+ await screen.rerender(
+ <Probe speech={{ ...speech, state: { ...speech.state, phase: "preparing" } }} />,
+ );
+ window.dispatchEvent(new KeyboardEvent("keyup", { code: "KeyD" }));
+ press();
+ window.dispatchEvent(new KeyboardEvent("keyup", { code: "KeyD" }));
+ finishStart();
+ await startup;
+ expect(speech.start).toHaveBeenCalledOnce();
+ expect(speech.stop).toHaveBeenCalledOnce();
+});
diff --git a/apps/web/src/speech/useDictationShortcut.ts b/apps/web/src/speech/useDictationShortcut.ts
new file mode 100644
index 00000000000..955bd9e5c4f
--- /dev/null
+++ b/apps/web/src/speech/useDictationShortcut.ts
@@ -0,0 +1,148 @@
+import type { VoiceInputState } from "@t3tools/client-runtime/voice-input";
+import type { ResolvedKeybindingsConfig } from "@t3tools/contracts";
+import { useEffect, useRef, type RefObject } from "react";
+
+import { isCommandPaletteOpen } from "../commandPaletteBus";
+import { useClientSettings } from "../hooks/useSettings";
+import { resolveShortcutCommand, shortcutLabelForCommand } from "../keybindings";
+import { getTerminalFocusOwner } from "../lib/terminalFocus";
+
+const HOLD_THRESHOLD_MS = 300;
+
+function shouldFinishDictationOnRelease(mode: "auto" | "hold" | "toggle", elapsedMs: number) {
+ return mode === "hold" || (mode === "auto" && elapsedMs >= HOLD_THRESHOLD_MS);
+}
+
+type SpeechActions = {
+ available: boolean;
+ state: VoiceInputState;
+ start(): Promise<void>;
+ stop(): Promise<void>;
+ cancel(): void;
+};
+
+type ShortcutInput = {
+ keybindings: ResolvedKeybindingsConfig;
+ speech: SpeechActions;
+ disabled: boolean;
+ terminalOpen: boolean;
+ modelPickerOpen: boolean;
+ /** When set, the chord targets this editor only while it has focus. */
+ targetRef?: RefObject<HTMLElement | null>;
+};
+
+// Several editors can bind the chord; the busy one, then the focused one, then the untargeted one wins.
+const shortcutTargets = new Map<symbol, () => ShortcutInput>();
+
+function ownsShortcut(id: symbol) {
+ const entries = [...shortcutTargets];
+ const busy = entries.find(([, read]) => {
+ const phase = read().speech.state.phase;
+ return phase !== "idle" && phase !== "error";
+ });
+ if (busy) return busy[0] === id;
+ const focused = entries.find(([, read]) =>
+ read().targetRef?.current?.contains(document.activeElement),
+ );
+ if (focused) return focused[0] === id;
+ return entries.find(([, read]) => !read().targetRef)?.[0] === id;
+}
+
+export function useDictationShortcut(input: ShortcutInput) {
+ const id = useRef(Symbol("dictation-target"));
+ const mode = useClientSettings((settings) => settings.voiceShortcutMode);
+ const current = useRef(input);
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed actions to the window keyboard listener and shortcut registry
+ useEffect(() => {
+ current.current = input;
+ });
+ const generation = useRef(0);
+ const pressed = useRef<{
+ generation: number;
+ code: string;
+ startedAt: number;
+ mode: typeof mode;
+ starting: Promise<void>;
+ } | null>(null);
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the window keyboard listeners and dictation target registry entry
+ useEffect(() => {
+ if (typeof window === "undefined") return;
+ let disposed = false;
+ shortcutTargets.set(id.current, () => current.current);
+ const finish = (event: KeyboardEvent | null) => {
+ const press = pressed.current;
+ if (!press || (event && (event.code || event.key) !== press.code)) return;
+ pressed.current = null;
+ if (shouldFinishDictationOnRelease(press.mode, performance.now() - press.startedAt)) {
+ void press.starting.then(() => {
+ if (!disposed && generation.current === press.generation)
+ return current.current.speech.stop();
+ });
+ }
+ };
+
+ const onKeyDown = (event: KeyboardEvent) => {
+ if (!ownsShortcut(id.current)) return;
+ if (event.isComposing || event.keyCode === 229) return;
+ const { speech } = current.current;
+ if (event.defaultPrevented || isCommandPaletteOpen()) return;
+ if (
+ event.key === "Escape" &&
+ (pressed.current !== null ||
+ (speech.state.phase !== "idle" && speech.state.phase !== "error"))
+ ) {
+ event.preventDefault();
+ event.stopImmediatePropagation();
+ pressed.current = null;
+ generation.current += 1;
+ speech.cancel();
+ return;
+ }
+ const { keybindings, terminalOpen, modelPickerOpen, disabled } = current.current;
+ const command = resolveShortcutCommand(event, keybindings, {
+ context: {
+ terminalFocus: getTerminalFocusOwner() !== null,
+ terminalOpen,
+ modelPickerOpen,
+ },
+ });
+ if (command !== "composer.dictation") return;
+ const canStart = speech.state.phase === "idle" || speech.state.phase === "error";
+ if (canStart && (!speech.available || disabled)) return;
+ event.preventDefault();
+ event.stopPropagation();
+ if (event.repeat || (!canStart && speech.state.phase !== "recording")) return;
+ // macOS may omit the previous non-modifier keyup while Command is held.
+ pressed.current = null;
+ generation.current += 1;
+ if (speech.state.phase === "recording") {
+ void speech.stop();
+ return;
+ }
+ pressed.current = {
+ generation: generation.current,
+ code: event.code || event.key,
+ startedAt: performance.now(),
+ mode,
+ starting: speech.start(),
+ };
+ };
+ const onBlur = () => finish(null);
+ window.addEventListener("keydown", onKeyDown, true);
+ window.addEventListener("keyup", finish, true);
+ window.addEventListener("blur", onBlur);
+ return () => {
+ disposed = true;
+ pressed.current = null;
+ shortcutTargets.delete(id.current);
+ window.removeEventListener("keydown", onKeyDown, true);
+ window.removeEventListener("keyup", finish, true);
+ window.removeEventListener("blur", onBlur);
+ };
+ }, [mode]);
+
+ return shortcutLabelForCommand(input.keybindings, "composer.dictation", {
+ context: { terminalFocus: false, terminalOpen: input.terminalOpen },
+ });
+}
diff --git a/apps/web/src/speech/useDictationSubmissionShortcut.ts b/apps/web/src/speech/useDictationSubmissionShortcut.ts
new file mode 100644
index 00000000000..5ee2b27d5b3
--- /dev/null
+++ b/apps/web/src/speech/useDictationSubmissionShortcut.ts
@@ -0,0 +1,33 @@
+import { useEffect, useRef, type RefObject } from "react";
+
+import { isCommandPaletteOpen } from "../commandPaletteBus";
+
+/** Keep submission shortcuts reachable when the frozen composer loses focus to the page. */
+export function useDictationSubmissionShortcut(input: {
+ enabled: boolean;
+ formRef: RefObject<HTMLFormElement | null>;
+ onKeyDown(event: KeyboardEvent): void;
+}) {
+ const latest = useRef(input);
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed inputs to the window keydown listener
+ useEffect(() => {
+ latest.current = input;
+ });
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to window keydown while dictation owns submission
+ useEffect(() => {
+ if (!input.enabled) return;
+ const onKeyDown = (event: KeyboardEvent) => {
+ if (event.defaultPrevented || isCommandPaletteOpen()) return;
+ const target = event.target;
+ if (!(target instanceof Element)) return;
+ if (target !== document.body && target !== document.documentElement) {
+ if (!latest.current.formRef.current?.contains(target)) return;
+ if (!target.closest('[data-testid="composer-editor"]')) return;
+ if (target.closest("textarea, input, button")) return;
+ }
+ latest.current.onKeyDown(event);
+ };
+ window.addEventListener("keydown", onKeyDown, true);
+ return () => window.removeEventListener("keydown", onKeyDown, true);
+ }, [input.enabled]);
+}
diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
new file mode 100644
index 00000000000..a41f359f68e
--- /dev/null
+++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
@@ -0,0 +1,493 @@
+import { useLayoutEffect } from "react";
+import { expect, it, vi } from "vite-plus/test";
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+import type { PreparedConnection } from "@t3tools/client-runtime/connection";
+import {
+ DEFAULT_CLIENT_SETTINGS,
+ DEFAULT_SERVER_SETTINGS,
+ type EnvironmentId,
+ type ProjectId,
+ type SpeechCustomWords,
+ type SpeechPostProcessingOptions,
+ type SpeechTranscriptionOptions,
+} from "@t3tools/contracts";
+
+import { act, render } from "../../test/render";
+import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
+import { chooseAudioDestination, chooseTranscriptCleanup, bindAudioGrant } from "./voiceConsent";
+import { writeBrowserClientSettings } from "../clientPersistenceStorage";
+
+const mocks = vi.hoisted(() => ({
+ voice: undefined as unknown as ReturnType<typeof useEnvironmentSpeechInput>,
+ settings: {} as Record<string, unknown>,
+ connected: new Set<string>(),
+ voiceCapable: new Set<string>(),
+ canDictate: new Set<string>(),
+ prepared: new Map<string, unknown>(),
+ primaryEnvironmentId: "primary-env",
+ statusRequests: [] as string[],
+ statusUnavailable: false,
+ modelState: "ready" as "ready" | "missing-model",
+ download: vi.fn<() => Promise<void>>(),
+ platforms: [] as string[],
+ microphoneRequests: [] as string[],
+ transcribeCalls: [] as string[],
+ cleanupCalls: [] as string[],
+ cleanupDraft: vi.fn(),
+ cleanupOptions: [] as SpeechPostProcessingOptions[],
+ recordingOptions: [] as SpeechTranscriptionOptions[],
+ projectWords: {} as Record<string, SpeechCustomWords>,
+ transcripts: [] as Array<string | Error>,
+ committed: vi.fn(),
+ draft: "",
+}));
+
+vi.mock("../state/session", () => ({
+ useEnvironmentScope: (id: string) => mocks.canDictate.has(id),
+ usePreparedConnection: (environmentId: string | null) => {
+ if (!environmentId || !mocks.connected.has(environmentId)) return Option.none();
+ let prepared = mocks.prepared.get(environmentId);
+ if (!prepared) {
+ prepared = { environmentId, label: environmentId };
+ mocks.prepared.set(environmentId, prepared);
+ }
+ return Option.some(prepared);
+ },
+}));
+vi.mock("../state/environments", () => ({
+ usePrimaryEnvironmentId: () => mocks.primaryEnvironmentId,
+ useEnvironment: (environmentId: string | null) =>
+ environmentId && mocks.connected.has(environmentId)
+ ? {
+ environmentId,
+ serverConfig: {
+ environment: {
+ capabilities: { voiceTranscription: mocks.voiceCapable.has(environmentId) },
+ },
+ },
+ }
+ : null,
+}));
+vi.mock("../hooks/useSettings", () => ({
+ useClientSettingsHydrated: () => true,
+ useClientSettings: (selector?: (settings: Record<string, unknown>) => unknown) => {
+ const settings = { ...DEFAULT_CLIENT_SETTINGS, ...mocks.settings };
+ return selector ? selector(settings) : settings;
+ },
+ useEnvironmentSettings: (
+ environmentId: string,
+ selector: (settings: Record<string, unknown>) => unknown,
+ ) =>
+ selector({
+ ...DEFAULT_SERVER_SETTINGS,
+ projectSettingsOverrides: Object.fromEntries(
+ Object.entries(environmentId === THREAD_ENV ? mocks.projectWords : {}).map(
+ ([projectId, words]) => [projectId, { speechProjectCustomWords: words }],
+ ),
+ ),
+ }),
+}));
+vi.mock("../lib/runtime", () => ({ runtime: { runPromise: Effect.runPromise } }));
+vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } }));
+vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@t3tools/client-runtime/voice-input")>()),
+ getEnvironmentSpeechStatus: (prepared: PreparedConnection) =>
+ Effect.sync(() => {
+ mocks.statusRequests.push(prepared.environmentId);
+ if (mocks.statusUnavailable) throw new Error("Temporarily offline");
+ return {
+ supported: true,
+ state: mocks.modelState,
+ modelId: "test-model",
+ model: "Test",
+ size: 1,
+ supportsStreaming: false,
+ supportsTranslation: false,
+ acceleration: "auto",
+ modelUnloadTimeout: "min_15",
+ gpuDevices: [],
+ };
+ }),
+ downloadEnvironmentSpeechModel: () => Effect.promise(mocks.download),
+ getEnvironmentSpeechModels: () => Effect.succeed({ models: [] }),
+ postProcessEnvironmentTranscript: (
+ prepared: PreparedConnection,
+ transcript: string,
+ draft: unknown,
+ options: SpeechPostProcessingOptions,
+ ) =>
+ Effect.sync(() => {
+ mocks.cleanupCalls.push(prepared.environmentId);
+ mocks.cleanupDraft(draft);
+ mocks.cleanupOptions.push(options);
+ return { text: `Cleaned ${transcript}` };
+ }),
+}));
+// Capture and the wire live behind the platform; its `prepared` is where audio would go.
+vi.mock("./browserVoiceInput", () => ({
+ createBrowserVoiceInputPlatform: (input: {
+ prepared: PreparedConnection;
+ getTranscriptionOptions?: () => SpeechTranscriptionOptions;
+ }) => {
+ const environmentId = input.prepared.environmentId;
+ mocks.platforms.push(environmentId);
+ return {
+ recorder: {
+ uri: `blob:${environmentId}`,
+ prepareToRecordAsync: async () => {
+ mocks.microphoneRequests.push(environmentId);
+ },
+ record() {},
+ stop: async () => {},
+ },
+ transcriber: {
+ prepare: async () => {
+ const options = input.getTranscriptionOptions?.();
+ if (options) mocks.recordingOptions.push(options);
+ return {
+ locale: "en",
+ transcribe: async () => {
+ mocks.transcribeCalls.push(environmentId);
+ const next = mocks.transcripts.shift() ?? "dictated";
+ if (next instanceof Error) throw next;
+ return next;
+ },
+ };
+ },
+ },
+ cancelRecording() {},
+ deleteRecording() {},
+ };
+ },
+}));
+
+const THREAD_ENV = "thread-env";
+
+function setup(options: { readonly destination?: string | null } = {}) {
+ mocks.settings = {};
+ setDestination(options.destination ?? null);
+ chooseTranscriptCleanup(false);
+ mocks.primaryEnvironmentId = "primary-env";
+ mocks.connected = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
+ mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
+ mocks.canDictate = new Set(mocks.voiceCapable);
+ mocks.prepared.clear();
+ mocks.statusRequests = [];
+ mocks.statusUnavailable = false;
+ mocks.modelState = "ready";
+ mocks.download.mockReset();
+ mocks.platforms = [];
+ mocks.microphoneRequests = [];
+ mocks.transcribeCalls = [];
+ mocks.cleanupCalls = [];
+ mocks.cleanupDraft.mockClear();
+ mocks.cleanupOptions = [];
+ mocks.recordingOptions = [];
+ mocks.projectWords = {};
+ mocks.transcripts = [];
+ mocks.committed.mockClear();
+ mocks.draft = "";
+}
+
+function setDestination(environmentId: string | null) {
+ chooseAudioDestination(
+ environmentId ? { kind: "environment", environmentId: environmentId as EnvironmentId } : null,
+ );
+}
+
+function Probe(props: {
+ readonly projectId?: string;
+ readonly projectName?: string;
+ readonly cleanup?: boolean;
+}) {
+ const value = useEnvironmentSpeechInput({
+ environmentId: THREAD_ENV as EnvironmentId,
+ projectId: props.projectId as ProjectId | undefined,
+ projectName: props.projectName,
+ cleanup: props.cleanup,
+ ownerKey: "draft",
+ draftText: mocks.draft,
+ readDraft: () => ({
+ text: mocks.draft,
+ selection: { start: mocks.draft.length, end: mocks.draft.length },
+ }),
+ commitDraft: mocks.committed,
+ });
+ useLayoutEffect(() => {
+ mocks.voice = value;
+ });
+ return null;
+}
+
+function expectNoAudioSent() {
+ expect(mocks.platforms).toEqual([]);
+ expect(mocks.microphoneRequests).toEqual([]);
+ expect(mocks.transcribeCalls).toEqual([]);
+ expect(mocks.statusRequests).toEqual([]);
+}
+
+it.each([
+ ["nothing is configured", {}],
+ // A transcription host picked by an earlier PoC build is not consent.
+ ["a legacy transcription host is configured", { voiceTranscriptionEnvironmentId: "env-a" }],
+])("stays local-only when %s, even with voice-capable environments", async (_, legacy) => {
+ setup();
+ mocks.settings = { ...legacy, speechSilenceFiltering: true };
+ await using _screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ expectNoAudioSent();
+ expect(mocks.committed).not.toHaveBeenCalled();
+ expect(mocks.voice.available).toBe(false);
+ expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
+});
+
+it.each([true, false])(
+ "sends audio with silence filtering %s only to the chosen environment",
+ async (speechSilenceFiltering) => {
+ setup({ destination: "env-a" });
+ mocks.settings = { speechSilenceFiltering };
+ await using _screen = await render(<Probe />);
+ expect(mocks.voice.transcriptionEnvironmentId).toBe("env-a");
+ await act(() => mocks.voice.start());
+ expect(mocks.voice.state.phase).toBe("recording");
+ await act(() => mocks.voice.stop());
+ expect(
+ new Set([...mocks.platforms, ...mocks.microphoneRequests, ...mocks.transcribeCalls]),
+ ).toEqual(new Set(["env-a"]));
+ expect(mocks.statusRequests.every((id) => id === "env-a")).toBe(true);
+ expect(mocks.recordingOptions[0]?.speechSilenceFiltering).toBe(speechSilenceFiltering);
+ // Cleanup is a separate opt-in and stays off by default.
+ expect(mocks.cleanupCalls).toEqual([]);
+ expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything());
+ },
+);
+
+it.each([
+ ["is disconnected", () => mocks.connected.delete("env-a")],
+ ["cannot operate", () => mocks.canDictate.delete("env-a")],
+ ["does not support transcription", () => mocks.voiceCapable.delete("env-a")],
+])(
+ "does not fall back to another environment when the chosen one %s",
+ async (_, breakDestination) => {
+ setup({ destination: "env-a" });
+ breakDestination();
+ await using _screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ expectNoAudioSent();
+ expect(mocks.voice.available).toBe(false);
+ },
+);
+
+it.each([
+ ["revoked", null],
+ ["moved to another environment", "env-b"],
+ ["renewed for the same environment", "env-a"],
+])("abandons an in-progress recording when consent is %s", async (_, next) => {
+ setup({ destination: "env-a" });
+ await using screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ expect(mocks.voice.state.phase).toBe("recording");
+ setDestination(next);
+ await screen.rerender(<Probe />);
+ expect(mocks.voice.state.phase).toBe("idle");
+ await act(() => mocks.voice.stop());
+ expect(mocks.transcribeCalls).toEqual([]);
+ expect(mocks.committed).not.toHaveBeenCalled();
+});
+
+it("retries a failed transcription on the same chosen environment", async () => {
+ setup({ destination: "env-a" });
+ mocks.transcripts = [new Error("inference failed"), "second try"];
+ await using _screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ await act(() => mocks.voice.stop());
+ expect(mocks.voice.state).toMatchObject({ phase: "error", errorAction: "retry" });
+ await act(() => mocks.voice.start());
+ if (mocks.voice.state.phase === "recording") await act(() => mocks.voice.stop());
+ expect(mocks.transcribeCalls).toEqual(["env-a", "env-a"]);
+ expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("second try", expect.anything());
+});
+
+it("reopens setup after reconnecting during a download and ignores its stale completion", async () => {
+ setup({ destination: "env-a" });
+ mocks.modelState = "missing-model";
+ let finishDownload!: () => void;
+ mocks.download.mockReturnValue(
+ new Promise<void>((resolve) => {
+ finishDownload = resolve;
+ }),
+ );
+ await using screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ let done: Promise<void> | undefined;
+ await act(() => {
+ done = mocks.voice.setup.download();
+ });
+ expect(mocks.voice.setup.downloading).toBe(true);
+ mocks.connected.delete("env-a");
+ await screen.rerender(<Probe />);
+ expect(mocks.voice.setup.open).toBe(false);
+ mocks.connected.add("env-a");
+ await screen.rerender(<Probe />);
+ await act(() => mocks.voice.start());
+ expect(mocks.voice.setup).toMatchObject({ open: true, downloading: false, step: 0 });
+ finishDownload();
+ await act(async () => {
+ await done;
+ });
+ expect(mocks.voice.setup.error).toBeNull();
+});
+
+it("does not resend a failed recording after consent is revoked", async () => {
+ setup({ destination: "env-a" });
+ mocks.transcripts = [new Error("inference failed")];
+ await using screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ await act(() => mocks.voice.stop());
+ expect(mocks.voice.state.phase).toBe("error");
+ setDestination(null);
+ await screen.rerender(<Probe />);
+ await act(() => mocks.voice.start());
+ expect(mocks.transcribeCalls).toEqual(["env-a"]);
+ expect(mocks.microphoneRequests).toEqual(["env-a"]);
+ expect(mocks.committed).not.toHaveBeenCalled();
+});
+
+it("revokes across windows even before the storage event and survives stale preference writes", async () => {
+ setup({ destination: "env-a" });
+ await using _screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ expect(mocks.voice.state.phase).toBe("recording");
+ const other = document.createElement("iframe");
+ document.body.append(other);
+ try {
+ const storage = other.contentWindow!.localStorage;
+ const key = "t3code:voice-audio-grant:v1";
+ const grant = JSON.parse(storage.getItem(key)!) as { id: string };
+ const canSend = bindAudioGrant(grant.id);
+ expect(canSend()).toBe(true);
+ storage.removeItem(key);
+ expect(canSend()).toBe(false);
+ writeBrowserClientSettings({ ...DEFAULT_CLIENT_SETTINGS, wordWrap: false });
+ await vi.waitFor(() => expect(mocks.voice.available).toBe(false));
+ expect(mocks.voice.state.phase).toBe("idle");
+ await act(() => mocks.voice.stop());
+ expect(mocks.transcribeCalls).toEqual([]);
+ setDestination("env-a");
+ expect(canSend()).toBe(false);
+ } finally {
+ other.remove();
+ }
+});
+
+it("cleans the transcript only after cleanup is separately enabled", async () => {
+ setup({ destination: "env-a" });
+ chooseTranscriptCleanup(true);
+ const nearby = "x".repeat(1_024);
+ mocks.draft = "unrelated private paragraph " + nearby;
+ await using screen = await render(<Probe />);
+ await act(() => mocks.voice.start());
+ await act(() => mocks.voice.stop());
+ expect(mocks.cleanupCalls).toHaveLength(1);
+ expect(mocks.cleanupDraft).toHaveBeenCalledExactlyOnceWith({
+ text: nearby,
+ selection: { start: nearby.length, end: nearby.length },
+ });
+ expect(mocks.committed).toHaveBeenCalledExactlyOnceWith(
+ mocks.draft + " Cleaned dictated",
+ expect.anything(),
+ );
+
+ await act(() => mocks.voice.start());
+ chooseTranscriptCleanup(false);
+ await screen.rerender(<Probe />);
+ await act(() => mocks.voice.stop());
+ expect(mocks.cleanupCalls).toHaveLength(1);
+});
+
+// The settings model test must show what the model heard, not what a provider rewrote.
+it.each([
+ { cleanup: false, authorized: true },
+ { cleanup: undefined, authorized: false },
+])(
+ "keeps a recording without authorized cleanup off the provider: $cleanup / $authorized",
+ async ({ cleanup, authorized }) => {
+ setup({ destination: "env-a" });
+ chooseTranscriptCleanup(true);
+ mocks.settings.speechCorrectionWord = "sorry";
+ if (!authorized) mocks.canDictate.delete(THREAD_ENV);
+ mocks.projectWords = { "proj-1": [{ term: "Acme", aliases: ["ack me"] }] };
+ await using _screen = await render(
+ <Probe {...(cleanup === undefined ? {} : { cleanup })} projectId="proj-1" />,
+ );
+ await act(() => mocks.voice.start());
+ await act(() => mocks.voice.stop());
+ expect(mocks.cleanupCalls).toEqual([]);
+ expect(mocks.recordingOptions.at(-1)?.speechCustomWords).toEqual(mocks.projectWords["proj-1"]);
+ expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBeUndefined();
+ expect(mocks.transcribeCalls).toEqual(["env-a"]);
+ expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything());
+ },
+);
+
+it("sends the project's vocabulary and name as hints, and the cue only with cleanup on", async () => {
+ setup({ destination: "env-a" });
+ mocks.settings.speechCustomWords = [{ term: "Effect", aliases: ["a fact"] }];
+ mocks.settings.speechCorrectionWord = "scratch that";
+ // Stored on the draft's environment under its project, never read from the audio destination.
+ mocks.projectWords = { "proj-1": [{ term: "effect", aliases: ["affect"] }] };
+ await using screen = await render(<Probe projectId="proj-1" projectName="Acme App" />);
+ await act(() => mocks.voice.start());
+ await act(() => mocks.voice.stop());
+ expect(mocks.recordingOptions.at(-1)?.speechCustomWords).toEqual([
+ { term: "Acme App", aliases: [] },
+ { term: "effect", aliases: ["affect"] },
+ ]);
+ expect(mocks.cleanupCalls).toEqual([]);
+
+ chooseTranscriptCleanup(true);
+ await screen.rerender(<Probe projectId="proj-1" projectName="Acme App" />);
+ await act(() => mocks.voice.start());
+ await act(() => mocks.voice.stop());
+ expect(mocks.recordingOptions.at(-1)?.speechCustomWords.map(({ term }) => term)).toEqual([
+ "scratch that",
+ "Acme App",
+ "effect",
+ ]);
+ expect(mocks.cleanupOptions.at(-1)).toMatchObject({ speechCorrectionWord: "scratch that" });
+ expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBe("scratch that");
+ chooseTranscriptCleanup(false);
+ await act(() => mocks.voice.start());
+ await act(() => mocks.voice.stop());
+ expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBeUndefined();
+ expect(mocks.recordingOptions.at(-1)?.speechCustomWords.map(({ term }) => term)).not.toContain(
+ "scratch that",
+ );
+});
+
+it("shares one status probe per destination across every mounted editor", async () => {
+ setup({ destination: "env-a" });
+ await using _screen = await render(
+ <>
+ <Probe />
+ <Probe />
+ <Probe />
+ </>,
+ );
+ expect(mocks.voice.available).toBe(true);
+ expect(mocks.statusRequests).toEqual(["env-a"]);
+});
+
+it("lets the user retry after a status failure without reselecting the destination", async () => {
+ setup({ destination: "env-a" });
+ mocks.statusUnavailable = true;
+ await using _screen = await render(<Probe />);
+ expect(mocks.statusRequests).toEqual(["env-a"]);
+ expect(mocks.voice.available).toBe(true);
+ expect(mocks.microphoneRequests).toEqual([]);
+ mocks.statusUnavailable = false;
+ await act(() => mocks.voice.start());
+ await act(() => mocks.voice.stop());
+ expect(mocks.transcribeCalls).toEqual(["env-a"]);
+});
diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts
new file mode 100644
index 00000000000..644edb303ae
--- /dev/null
+++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts
@@ -0,0 +1,552 @@
+import {
+ cancelEnvironmentSpeechModelDownload,
+ downloadEnvironmentSpeechModel,
+ getEnvironmentSpeechModels,
+ getEnvironmentSpeechStatus,
+ postProcessEnvironmentTranscript,
+ VoiceInputController,
+ voiceInputBlocksSubmission,
+ type VoiceDraftSnapshot,
+ type VoiceInputControllerDependencies,
+ type VoiceInputState,
+} from "@t3tools/client-runtime/voice-input";
+import type { PreparedConnection } from "@t3tools/client-runtime/connection";
+import { AuthOrchestrationOperateScope } from "@t3tools/contracts";
+import type {
+ EnvironmentId,
+ EnvironmentSpeechModel,
+ EnvironmentSpeechStatus,
+ ProjectId,
+ SpeechPostProcessingOptions,
+ SpeechStreamText,
+ SpeechTranscriptionOptions,
+} from "@t3tools/contracts";
+import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
+import { resolveSpeechVocabulary } from "@t3tools/shared/speech";
+import * as Option from "effect/Option";
+import { useEffect, useRef, useState } from "react";
+
+import { useStableCallback } from "~/hooks/useStableCallback";
+import { toastManager } from "../components/ui/toast";
+import {
+ useClientSettings,
+ useClientSettingsHydrated,
+ useEnvironmentSettings,
+} from "../hooks/useSettings";
+import { runtime } from "../lib/runtime";
+import { useEnvironment } from "../state/environments";
+import { useEnvironmentScope, usePreparedConnection } from "../state/session";
+import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
+import { bindAudioGrant, readCleanupGrant, useAudioGrant } from "./voiceConsent";
+
+const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
+const WAITING_STATE: VoiceInputState = { phase: "preparing", error: null, errorAction: null };
+const INITIAL_SETUP = {
+ open: false,
+ step: 0,
+ model: null as EnvironmentSpeechModel | null,
+ downloading: false,
+ error: null as string | null,
+};
+
+// Every mounted editor shares one status probe per destination; starting a recording refreshes it.
+const statusProbes = new WeakMap<PreparedConnection, Promise<EnvironmentSpeechStatus>>();
+function probeSpeechStatus(prepared: PreparedConnection, refresh = false) {
+ const cached = refresh ? undefined : statusProbes.get(prepared);
+ if (cached) return cached;
+ const probe = runtime.runPromise(getEnvironmentSpeechStatus(prepared));
+ statusProbes.set(prepared, probe);
+ probe.catch(() => {
+ if (statusProbes.get(prepared) === probe) statusProbes.delete(prepared);
+ });
+ return probe;
+}
+
+type DraftInput = {
+ readonly text: string;
+ readonly selection: { readonly start: number; readonly end: number };
+};
+
+function cleanupContext({ text, selection }: DraftInput): DraftInput {
+ const limit = 1_024;
+ const start = Math.max(0, Math.min(selection.start, text.length));
+ const end = Math.max(start, Math.min(selection.end, text.length));
+ const before = text.slice(Math.max(0, start - limit), start);
+ const selected = text.slice(start, Math.min(end, start + limit));
+ return {
+ text: before + selected + text.slice(end, end + limit),
+ selection: { start: before.length, end: before.length + selected.length },
+ };
+}
+
+type HookInput = {
+ /** The draft's own environment. Optional transcript cleanup runs there, never on the audio destination. */
+ readonly environmentId: EnvironmentId;
+ /** The draft's project: its vocabulary on that environment and its name become recognition hints. */
+ readonly projectId?: ProjectId | null | undefined;
+ readonly projectName?: string | null | undefined;
+ readonly ownerKey: string;
+ readonly draftText: string;
+ readonly disabled?: boolean | undefined;
+ /** False keeps the transcript off the cleanup provider even when cleanup is on; the model test uses it. */
+ readonly cleanup?: boolean | undefined;
+ readonly readDraft: () => DraftInput;
+ readonly commitDraft: (
+ text: string,
+ selection: { readonly start: number; readonly end: number },
+ ) => void;
+};
+
+/**
+ * Microphone audio leaves this device only for the environment the person chose in
+ * Voice settings. Nothing is inferred from the primary or the draft's environment, and
+ * the recording is abandoned the moment that choice changes or its connection drops.
+ */
+export function useVoiceTranscriptionDestination() {
+ const hydrated = useClientSettingsHydrated();
+ const grant = useAudioGrant();
+ const environmentId = hydrated && grant ? grant.destination.environmentId : null;
+ const environment = useEnvironment(environmentId);
+ const connection = Option.getOrNull(usePreparedConnection(environmentId));
+ const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope);
+ const supported = environment?.serverConfig?.environment.capabilities.voiceTranscription === true;
+ return {
+ environmentId,
+ grantId: grant?.id ?? null,
+ label: environment?.label ?? null,
+ prepared: supported && canDictate ? connection : null,
+ };
+}
+
+export function useEnvironmentSpeechInput(input: HookInput) {
+ const destination = useVoiceTranscriptionDestination();
+ const prepared = destination.prepared;
+ const grantId = destination.grantId;
+ const originatingEnvironment = useEnvironment(input.environmentId);
+ const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId));
+ const canCleanUp = useEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope);
+ const cleanupPrepared =
+ canCleanUp &&
+ originatingEnvironment?.serverConfig?.environment.capabilities.voiceTranscription === true
+ ? originatingConnection
+ : null;
+ const clientSettings = useClientSettings();
+ const projectId = input.projectId ?? null;
+ const projectWords = useEnvironmentSettings(
+ input.environmentId,
+ (settings) => resolveProjectSettings(settings, projectId).settings.speechProjectCustomWords,
+ );
+ const [status, setStatus] = useState<{
+ readonly prepared: PreparedConnection;
+ readonly value: EnvironmentSpeechStatus | null;
+ } | null>(null);
+ const [controllerState, setControllerState] = useState({
+ prepared,
+ grantId,
+ value: INITIAL_STATE,
+ });
+ const [queuedStart, setQueuedStart] = useState<PreparedConnection | null>(null);
+ const [level, setLevel] = useState(0);
+ const [preview, setPreview] = useState<SpeechStreamText | null>(null);
+ const [setup, setSetup] = useState(INITIAL_SETUP);
+ const [previousOwner, setPreviousOwner] = useState({
+ key: input.ownerKey,
+ disabled: input.disabled,
+ });
+ if (controllerState.prepared !== prepared || controllerState.grantId !== grantId) {
+ setControllerState({ prepared, grantId, value: INITIAL_STATE });
+ setSetup(INITIAL_SETUP);
+ setQueuedStart(null);
+ setPreview(null);
+ setLevel(0);
+ }
+ if (previousOwner.key !== input.ownerKey || previousOwner.disabled !== input.disabled) {
+ setPreviousOwner({ key: input.ownerKey, disabled: input.disabled });
+ setQueuedStart(null);
+ setSetup(INITIAL_SETUP);
+ }
+ const state: VoiceInputState =
+ queuedStart === prepared && prepared
+ ? WAITING_STATE
+ : controllerState.prepared === prepared
+ ? controllerState.value
+ : INITIAL_STATE;
+
+ const controllerRef = useRef<VoiceInputController | null>(null);
+ const startRequestRef = useRef(0);
+ const pendingSubmissionRef = useRef<{
+ ownerKey: string;
+ submit: (text: string) => void;
+ text?: string;
+ } | null>(null);
+ const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords });
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props and settings to the asynchronous recording controller
+ useEffect(() => {
+ latestRef.current = { input, clientSettings, cleanupPrepared, projectWords };
+ });
+ const draftRevisionRef = useRef({ ownerKey: input.ownerKey, text: input.draftText, revision: 0 });
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tracks committed draft revisions so a late transcript cannot overwrite an edited draft
+ useEffect(() => {
+ const revision = draftRevisionRef.current;
+ if (revision.ownerKey !== input.ownerKey || revision.text !== input.draftText) {
+ draftRevisionRef.current = {
+ ownerKey: input.ownerKey,
+ text: input.draftText,
+ revision: revision.revision + 1,
+ };
+ }
+ }, [input.ownerKey, input.draftText]);
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the recording controller bound to the consented destination; it is disposed with the consent
+ useEffect(() => {
+ if (!prepared || !grantId || typeof navigator === "undefined") return;
+ const hasAudioConsent = bindAudioGrant(grantId);
+ let disposed = false;
+ let controller: VoiceInputController;
+ let recordingPreferences:
+ | {
+ readonly cleanup: SpeechPostProcessingOptions | null;
+ readonly cleanupGrant: string | null;
+ readonly environmentId: EnvironmentId;
+ }
+ | undefined;
+
+ const readDraft = (): VoiceDraftSnapshot => {
+ const current = latestRef.current.input;
+ const draft = current.readDraft();
+ const revision = draftRevisionRef.current;
+ if (revision.ownerKey !== current.ownerKey || revision.text !== draft.text) {
+ draftRevisionRef.current = {
+ ownerKey: current.ownerKey,
+ text: draft.text,
+ revision: revision.revision + 1,
+ };
+ }
+ return {
+ ownerKey: current.ownerKey,
+ text: draft.text,
+ selection: draft.selection,
+ revision: draftRevisionRef.current.revision,
+ };
+ };
+ const platform = createBrowserVoiceInputPlatform({
+ prepared,
+ hasAudioConsent,
+ getTranscriptionOptions: (): SpeechTranscriptionOptions => {
+ const {
+ clientSettings: settings,
+ projectWords: project,
+ input: current,
+ } = latestRef.current;
+ const cleanupGrant = current.cleanup === false ? null : readCleanupGrant();
+ const cleanupEnabled = cleanupGrant !== null;
+ const cue =
+ cleanupEnabled && latestRef.current.cleanupPrepared ? settings.speechCorrectionWord : "";
+ const speechCustomWords = resolveSpeechVocabulary({
+ personal: settings.speechCustomWords,
+ project,
+ hints: [cue, current.projectName ?? ""],
+ });
+ recordingPreferences = {
+ cleanupGrant,
+ environmentId: current.environmentId,
+ cleanup: cleanupEnabled
+ ? {
+ speechCustomWords,
+ speechPostProcessingPrompt: settings.speechPostProcessingPrompt,
+ ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
+ }
+ : null,
+ };
+ return {
+ speechLanguage: settings.speechLanguage,
+ speechTranslateToEnglish: settings.speechTranslateToEnglish,
+ speechCustomWords,
+ speechRemoveFillerWords: settings.speechRemoveFillerWords,
+ ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
+ speechSilenceFiltering: settings.speechSilenceFiltering,
+ };
+ },
+ getMicrophoneId: () => latestRef.current.clientSettings.voiceMicrophone,
+ onLevel: setLevel,
+ onDurationLimit: () => void controller.stop(),
+ onText: (text) => {
+ if (!disposed) setPreview(text);
+ },
+ onError: (message) => {
+ if (!disposed) void controller.interruptRecording(message);
+ },
+ });
+ const postProcess: NonNullable<VoiceInputControllerDependencies["postProcess"]> = (
+ transcript,
+ options,
+ ) => {
+ const preferences = recordingPreferences;
+ const { input: current, cleanupPrepared: target } = latestRef.current;
+ if (
+ !preferences?.cleanup ||
+ !hasAudioConsent() ||
+ preferences.cleanupGrant !== readCleanupGrant() ||
+ current.cleanup === false ||
+ current.environmentId !== preferences.environmentId ||
+ current.ownerKey !== options.draft.ownerKey
+ )
+ return null;
+ if (!target)
+ return Promise.reject(
+ new Error("The draft's environment is unavailable for transcript cleanup."),
+ );
+ return runtime
+ .runPromise(
+ postProcessEnvironmentTranscript(
+ target,
+ transcript,
+ cleanupContext(options.draft),
+ preferences.cleanup,
+ () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
+ ),
+ options,
+ )
+ .then((result) => result.text);
+ };
+ controller = new VoiceInputController({
+ retryFailedRecording: true,
+ recorder: platform.recorder,
+ getTranscriber: () => platform.transcriber,
+ requestPermission: async () => ({ granted: true, canAskAgain: true }),
+ configureRecording: async () => undefined,
+ releaseRecording: async () => platform.cancelRecording(),
+ deleteRecording: platform.deleteRecording,
+ postProcess,
+ onPostProcessingError: () =>
+ toastManager.add({
+ type: "warning",
+ title: "Transcript cleanup failed",
+ description: "The original transcript was added.",
+ }),
+ readDraft,
+ commitDraft: (text, selection) => {
+ if (latestRef.current.input.disabled) return;
+ latestRef.current.input.commitDraft(text, selection);
+ if (pendingSubmissionRef.current) pendingSubmissionRef.current.text = text;
+ },
+ onStateChange: (value) => {
+ if (disposed) return;
+ if (value.phase === "error") pendingSubmissionRef.current = null;
+ setControllerState({ prepared, grantId, value });
+ if (value.phase !== "recording" && value.phase !== "transcribing") setPreview(null);
+ },
+ });
+ controllerRef.current = controller;
+ return () => {
+ disposed = true;
+ pendingSubmissionRef.current = null;
+ startRequestRef.current += 1;
+ setQueuedStart(null);
+ controller.dispose();
+ if (controllerRef.current === controller) controllerRef.current = null;
+ };
+ }, [prepared, grantId]);
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available
+ useEffect(() => {
+ if (!prepared) return;
+ let disposed = false;
+ void probeSpeechStatus(prepared)
+ .then((value) => {
+ if (!disposed) setStatus({ prepared, value });
+ })
+ .catch(() => {
+ if (!disposed) setStatus({ prepared, value: null });
+ });
+ return () => {
+ disposed = true;
+ };
+ }, [prepared]);
+ const currentStatus = status?.prepared === prepared ? status.value : null;
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download
+ useEffect(() => {
+ if (!setup.open || !setup.downloading || !prepared || !currentStatus?.supported) return;
+ const modelId = currentStatus.modelId;
+ let disposed = false;
+ let refreshing = false;
+ const timer = window.setInterval(() => {
+ if (refreshing) return;
+ refreshing = true;
+ void runtime
+ .runPromise(getEnvironmentSpeechModels(prepared))
+ .then((result) => {
+ if (disposed) return;
+ const model = result.models.find((candidate) => candidate.id === modelId) ?? null;
+ setSetup((current) => ({ ...current, model }));
+ })
+ .catch(() => undefined)
+ .finally(() => {
+ refreshing = false;
+ });
+ }, 1_000);
+ return () => {
+ disposed = true;
+ window.clearInterval(timer);
+ };
+ }, [currentStatus, prepared, setup.downloading, setup.open]);
+
+ const previousOwnerRef = useRef(input.ownerKey);
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons the recording and any queued submission when the draft's owner changes
+ useEffect(() => {
+ if (previousOwnerRef.current === input.ownerKey && !input.disabled) return;
+ previousOwnerRef.current = input.ownerKey;
+ pendingSubmissionRef.current = null;
+ startRequestRef.current += 1;
+ controllerRef.current?.ownerChanged();
+ }, [input.ownerKey, input.disabled]);
+
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- delivers a queued submission only after the transcript has committed to the draft
+ useEffect(() => {
+ const pending = pendingSubmissionRef.current;
+ if (state.phase !== "idle" || !pending) return;
+ pendingSubmissionRef.current = null;
+ if (!input.disabled && pending.ownerKey === input.ownerKey && pending.text !== undefined)
+ pending.submit(pending.text);
+ }, [state, input.ownerKey, input.disabled]);
+
+ const submitAfterDictation = useStableCallback((submit: (text: string) => void) => {
+ if (latestRef.current.input.disabled || state.phase === "preparing") return;
+ const controller = controllerRef.current;
+ if (!controller || !voiceInputBlocksSubmission(state)) {
+ submit(latestRef.current.input.readDraft().text);
+ return;
+ }
+ if (pendingSubmissionRef.current) return;
+ pendingSubmissionRef.current = { ownerKey: latestRef.current.input.ownerKey, submit };
+ if (controller.currentState.phase === "recording") void controller.stop();
+ });
+
+ const start = useStableCallback(async () => {
+ const request = ++startRequestRef.current;
+ const controller = controllerRef.current;
+ const owner = latestRef.current.input.ownerKey;
+ if (latestRef.current.input.disabled || !controller || !prepared) return;
+ const stillCurrent = () =>
+ request === startRequestRef.current &&
+ controllerRef.current === controller &&
+ latestRef.current.input.ownerKey === owner &&
+ !latestRef.current.input.disabled;
+ setQueuedStart(prepared);
+ let latestStatus: EnvironmentSpeechStatus;
+ try {
+ latestStatus = await probeSpeechStatus(prepared, true);
+ } catch (error) {
+ if (!stillCurrent()) return;
+ setQueuedStart(null);
+ toastManager.add({
+ type: "error",
+ title: "Could not start voice input",
+ description: error instanceof Error ? error.message : String(error),
+ });
+ return;
+ }
+ if (!stillCurrent()) return;
+ setQueuedStart(null);
+ setStatus({ prepared, value: latestStatus });
+ if (!latestStatus.supported) {
+ toastManager.add({
+ type: "error",
+ title: "Voice input is not available",
+ description: latestStatus.reason,
+ });
+ return;
+ }
+ if (latestStatus.state === "missing-model") {
+ setSetup({ ...INITIAL_SETUP, open: true });
+ return;
+ }
+ setLevel(0);
+ await controller.start();
+ });
+
+ const downloadSetupModel = useStableCallback(async () => {
+ if (!prepared || !currentStatus?.supported || setup.downloading) return;
+ const request = ++startRequestRef.current;
+ setSetup((current) => ({ ...current, downloading: true, error: null }));
+ try {
+ await runtime.runPromise(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));
+ if (request !== startRequestRef.current) return;
+ const next = await probeSpeechStatus(prepared, true);
+ if (request !== startRequestRef.current) return;
+ setStatus({ prepared, value: next });
+ setSetup((current) => ({
+ ...current,
+ downloading: false,
+ step: next.supported && next.state !== "missing-model" ? 1 : current.step,
+ error:
+ next.supported && next.state !== "missing-model"
+ ? null
+ : "The model is not ready. Try downloading it again.",
+ }));
+ } catch {
+ if (request !== startRequestRef.current) return;
+ setSetup((current) => ({
+ ...current,
+ downloading: false,
+ error: "Could not download the model. Check this environment's connection and try again.",
+ }));
+ }
+ });
+
+ const cancelSetupDownload = useStableCallback(async () => {
+ if (!prepared || !currentStatus?.supported) return;
+ startRequestRef.current += 1;
+ setSetup((current) => ({ ...current, open: false, downloading: false }));
+ try {
+ await runtime.runPromise(
+ cancelEnvironmentSpeechModelDownload(prepared, currentStatus.modelId),
+ );
+ } catch (error) {
+ setSetup((current) => ({
+ ...current,
+ error: error instanceof Error ? error.message : String(error),
+ }));
+ }
+ });
+
+ return {
+ available:
+ prepared !== null &&
+ status?.prepared === prepared &&
+ status.value?.supported !== false &&
+ typeof navigator !== "undefined" &&
+ Boolean(navigator.mediaDevices?.getUserMedia) &&
+ typeof AudioWorkletNode !== "undefined",
+ status: currentStatus,
+ transcriptionEnvironmentId: destination.environmentId,
+ transcriptionEnvironmentLabel: destination.label,
+ setup: {
+ ...setup,
+ setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
+ download: downloadSetupModel,
+ cancelDownload: cancelSetupDownload,
+ startRecording: async () => {
+ setSetup((current) => ({ ...current, open: false }));
+ await start();
+ },
+ },
+ state,
+ preview: state.phase === "recording" || state.phase === "transcribing" ? preview : null,
+ level,
+ preparing: state.phase === "preparing",
+ blocksSubmission: voiceInputBlocksSubmission(state),
+ freezesEditor: voiceInputBlocksSubmission(state),
+ start,
+ submitAfterDictation,
+ stop: useStableCallback(() => controllerRef.current?.stop() ?? Promise.resolve()),
+ cancel: useStableCallback(() => {
+ pendingSubmissionRef.current = null;
+ startRequestRef.current += 1;
+ setQueuedStart(null);
+ controllerRef.current?.cancel();
+ }),
+ skipPostProcessing: useStableCallback(() => controllerRef.current?.skipPostProcessing()),
+ };
+}
diff --git a/apps/web/src/speech/useTextFieldSpeech.tsx b/apps/web/src/speech/useTextFieldSpeech.tsx
new file mode 100644
index 00000000000..5486482d623
--- /dev/null
+++ b/apps/web/src/speech/useTextFieldSpeech.tsx
@@ -0,0 +1,131 @@
+import { useAtomValue } from "@effect/atom-react";
+import type { EnvironmentId, ProjectId } from "@t3tools/contracts";
+import { useEffect, useRef, type ReactNode, type RefObject } from "react";
+
+import {
+ ComposerSpeechButton,
+ ComposerSpeechRecordingPill,
+} from "~/components/chat/ComposerSpeechButton";
+import { VoiceInputSetup } from "~/components/chat/VoiceInputSetup";
+import { primaryServerKeybindingsAtom } from "~/state/server";
+import { useDictationShortcut } from "./useDictationShortcut";
+import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
+
+/** Binds dictation to a plain textarea draft; the shortcut targets it only while it has focus. */
+export function useTextFieldSpeech(input: {
+ readonly environmentId: EnvironmentId;
+ readonly projectId?: ProjectId | null | undefined;
+ readonly ownerKey: string;
+ readonly text: string;
+ readonly textareaRef: RefObject<HTMLTextAreaElement | null>;
+ readonly onTextChange: (text: string) => void;
+ readonly disabled?: boolean | undefined;
+}) {
+ const latest = useRef(input);
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed field callbacks to the asynchronous recording controller
+ useEffect(() => {
+ latest.current = input;
+ });
+ const keybindings = useAtomValue(primaryServerKeybindingsAtom);
+ const speech = useEnvironmentSpeechInput({
+ environmentId: input.environmentId,
+ projectId: input.projectId,
+ ownerKey: input.ownerKey,
+ draftText: input.text,
+ disabled: input.disabled,
+ readDraft: () => {
+ const field = latest.current.textareaRef.current;
+ const cursor = field?.selectionStart ?? latest.current.text.length;
+ return {
+ text: field?.value ?? latest.current.text,
+ selection: { start: cursor, end: field?.selectionEnd ?? cursor },
+ };
+ },
+ commitDraft: (text, selection) => {
+ latest.current.onTextChange(text);
+ window.requestAnimationFrame(() => {
+ const field = latest.current.textareaRef.current;
+ if (!field) return;
+ field.focus({ preventScroll: true });
+ field.setSelectionRange(selection.start, selection.end);
+ });
+ },
+ });
+ const shortcutLabel = useDictationShortcut({
+ keybindings,
+ speech,
+ disabled: input.disabled ?? false,
+ terminalOpen: false,
+ modelPickerOpen: false,
+ targetRef: input.textareaRef,
+ });
+ return { ...speech, shortcutLabel };
+}
+
+export type TextFieldSpeech = ReturnType<typeof useTextFieldSpeech>;
+
+export function TextFieldSpeechControls({
+ speech,
+ disabled = false,
+}: {
+ speech: TextFieldSpeech;
+ disabled?: boolean;
+}) {
+ return (
+ <>
+ {speech.preview ? (
+ <p role="status" className="w-full text-sm text-muted-foreground">
+ {speech.preview.committed}
+ <span className="opacity-70">{speech.preview.tentative}</span>
+ </p>
+ ) : null}
+ <div className="flex min-w-0 items-center gap-1">
+ <ComposerSpeechRecordingPill
+ state={speech.state}
+ progress={null}
+ level={speech.level}
+ finishShortcutLabel={speech.shortcutLabel}
+ cancelShortcutLabel="Esc"
+ disabled={disabled}
+ onStart={() => void speech.start()}
+ onStop={() => void speech.stop()}
+ onCancel={speech.cancel}
+ onSkipPostProcessing={speech.skipPostProcessing}
+ />
+ {speech.available ? (
+ <ComposerSpeechButton
+ state={speech.state}
+ progress={null}
+ shortcutLabel={speech.shortcutLabel}
+ disabled={disabled}
+ onStart={() => void speech.start()}
+ onCancel={speech.cancel}
+ />
+ ) : null}
+ </div>
+ <VoiceInputSetup
+ environmentId={speech.transcriptionEnvironmentId}
+ open={speech.setup.open}
+ step={speech.setup.step}
+ status={speech.status}
+ model={speech.setup.model}
+ downloading={speech.setup.downloading}
+ error={speech.setup.error}
+ onOpenChange={speech.setup.setOpen}
+ onDownload={() => void speech.setup.download()}
+ onCancelDownload={() => void speech.setup.cancelDownload()}
+ onStartRecording={() => void speech.setup.startRecording()}
+ />
+ </>
+ );
+}
+
+/** Render-prop form for editors that only sometimes have an environment to dictate on. */
+export function SpeechTextField({
+ children,
+ ...input
+}: Parameters<typeof useTextFieldSpeech>[0] & {
+ children: (speech: TextFieldSpeech) => ReactNode;
+}) {
+ return children(useTextFieldSpeech(input));
+}
diff --git a/apps/web/src/speech/voiceConsent.ts b/apps/web/src/speech/voiceConsent.ts
new file mode 100644
index 00000000000..81da7cd6adb
--- /dev/null
+++ b/apps/web/src/speech/voiceConsent.ts
@@ -0,0 +1,57 @@
+import { VoiceTranscriptionDestination } from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
+import { randomUUID } from "../lib/utils";
+import {
+ readLocalStorageValue,
+ useLocalStorage,
+ writeLocalStorageValue,
+} from "../hooks/useLocalStorage";
+
+// Separate keys: a stale window saving ordinary preferences must not restore either grant.
+const AUDIO_KEY = "t3code:voice-audio-grant:v1";
+const CLEANUP_KEY = "t3code:voice-cleanup-grant:v1";
+const AudioGrant = Schema.NullOr(
+ Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination }),
+);
+const CleanupGrant = Schema.NullOr(Schema.String);
+const decodeAudioGrant = Schema.decodeSync(Schema.fromJsonString(AudioGrant));
+
+export function useAudioGrant() {
+ return useLocalStorage(AUDIO_KEY, null, AudioGrant)[0];
+}
+
+export function chooseAudioDestination(destination: VoiceTranscriptionDestination | null) {
+ writeLocalStorageValue(
+ AUDIO_KEY,
+ destination && { id: randomUUID(), destination },
+ null,
+ AudioGrant,
+ );
+}
+
+// Bind this recording to this act of consent, even if the user later reselects the same destination.
+export function bindAudioGrant(id: string) {
+ try {
+ const raw = window.localStorage.getItem(AUDIO_KEY);
+ if (!raw || decodeAudioGrant(raw)?.id !== id) return () => false;
+ return () => {
+ try {
+ return window.localStorage.getItem(AUDIO_KEY) === raw;
+ } catch {
+ return false;
+ }
+ };
+ } catch {
+ return () => false;
+ }
+}
+
+export function useCleanupGrant() {
+ return useLocalStorage(CLEANUP_KEY, null, CleanupGrant)[0];
+}
+export function readCleanupGrant() {
+ return readLocalStorageValue(CLEANUP_KEY, null, CleanupGrant);
+}
+export function chooseTranscriptCleanup(enabled: boolean) {
+ writeLocalStorageValue(CLEANUP_KEY, enabled ? randomUUID() : null, null, CleanupGrant);
+}
diff --git a/docs/README.md b/docs/README.md
index 99b36dcf109..7ee82b0232f 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -11,6 +11,7 @@
- [Project settings](./user/project-settings.md)
- [Appearance and themes](./user/appearance.md)
- [Keyboard shortcuts](./user/keybindings.md)
+- [Voice input](./user/voice-input.md)
- [SnapShots](./user/snap-shot.md)
- [Visual replies](./user/html-renders.md)
- [Import browser sessions](./user/browser-import.md)
diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md
index 55e3007a487..84cfeaea373 100644
--- a/docs/internals/voice-input.md
+++ b/docs/internals/voice-input.md
@@ -2,14 +2,55 @@
Transcription edits a composer draft. It does not submit an agent turn. Audio is
temporary client input, and only normal message submission sends the resulting
-text. The current implementation transcribes locally on supported iOS devices;
-environment-backed transcription is not implemented.
+text. Mobile transcribes on the device; web and desktop send audio to one
+environment the person chose.
The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts)
owns the operation while the client supplies capture and transcription. Preparation
binds the transcriber and resolved locale for the whole recording. Draft ownership,
text, and revision are captured before recording and checked before insertion, so
-a late transcript cannot overwrite a draft that was edited or replaced.
+a late transcript cannot overwrite a draft that was edited or replaced. A
+transcriber may expose a streaming session, in which case audio leaves during
+capture and `finish` replaces the file-based `transcribe`.
+
+## Where audio may go
+
+Only an explicit [audio grant](../../apps/web/src/speech/voiceConsent.ts) authorizes
+off-device transcription. It lives outside the preferences snapshot so another
+window saving unrelated settings cannot resurrect a revoked grant. Each new
+choice gets a distinct grant, even if the destination was previously selected. The
+[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds a
+recording controller to the prepared connection of that destination and to
+nothing else. When the destination changes, disconnects, or stops advertising the
+`voiceTranscription` capability, the controller is disposed: a recording in
+progress is discarded, a failed recording kept for retry is deleted, and nothing
+falls back to another environment. Every sender in the web app, including the
+settings page's transcription test, goes through that hook. The microphone loopback
+test captures separately for local playback and sends nothing. The transport
+refuses redirects so credentials and audio cannot follow one to another host.
+
+Transcript cleanup requires its own grant, off by default.
+It sends text, never audio, to the text generation provider of the draft's
+own environment, because that is where the provider credentials live. A later
+backend adds its own destination kind, disclosure, and transcriber; capture and
+the controller stay as they are.
+
+## Server side
+
+The [speech service](../../apps/server/src/speech/SpeechService.ts) runs
+transcribe-cpp in a child process so a hung native call can be killed without
+taking the server down; see [native.ts](../../apps/server/src/speech/native.ts)
+for the packaging constraints (Node SEA, Electron ASAR). One native request runs
+at a time. Batch uploads carry their options in the body framing defined in
+[shared/speech.ts](../../packages/shared/src/speech.ts); streaming sends them in
+the socket's first command. Model files and the selected model, hardware
+placement, and unload timeout are environment state in server settings.
+
+The Silero silence detector is optional. If its download fails, recordings use
+raw audio during the retry cooldown so a detector outage cannot block dictation.
+Explicitly installing a speech model also retries the detector without waiting.
+
+## Cancellation and resource ownership
Cancellation invalidates a result immediately, but resources stay owned until the
underlying work settles. Apple's native transcription call cannot be interrupted
@@ -17,4 +58,6 @@ once started. Releasing the session or deleting its recording when the abort sig
fires would race that work. The [transcription contract](../../packages/client-runtime/src/voice-input/transcription.ts)
therefore requires implementations to settle only after their work has stopped;
the [Apple binding](../../apps/mobile/src/native/voiceTranscription.ios.ts) checks
-cancellation between native calls and discards late results.
+cancellation between native calls and discards late results. On the server,
+interrupting a batch request kills the worker process instead; the next request
+reloads the model.
diff --git a/docs/user/composer.md b/docs/user/composer.md
index 86dff30a79c..6ab88afb3bd 100644
--- a/docs/user/composer.md
+++ b/docs/user/composer.md
@@ -184,6 +184,12 @@ Stashes containing uploaded files must be restored in their original environment
Those files are retained for 24 hours. After an upload expires, restore the prompt
and use **Attach again** or remove the missing file before sending.
+## Voice input on web and desktop
+
+Use the composer's microphone or `mod+shift+d` to dictate into your draft. Audio
+is transcribed on an environment you choose in **Settings → Voice**; nothing is
+sent until you choose one. See [Voice input](./voice-input.md).
+
## Voice input on mobile
On supported iPhones with iOS 26 or later and Android devices with Android 12 or
diff --git a/docs/user/keybindings.md b/docs/user/keybindings.md
index 3654c0b3b04..1c558bd0d44 100644
--- a/docs/user/keybindings.md
+++ b/docs/user/keybindings.md
@@ -5,6 +5,11 @@ also lists the command IDs and defaults available in your version.
## Composer controls
+`mod+shift+d` starts dictation in the focused editor, or the composer, and finishes it on the next press;
+`Esc` discards the recording. Change the chord under **Composer: Start or Finish
+Dictation**, and the hold or toggle behavior in **Settings → Voice**. See
+[Voice input](./voice-input.md).
+
In **Settings → Chat → Send shortcut**, choose whether Enter sends, requires
`mod+Enter` for multiline prompts, or always requires `mod+Enter`. `Shift+Enter`
inserts a new line. This applies to the web and desktop composer at desktop widths.
diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md
new file mode 100644
index 00000000000..33da1b4ac3a
--- /dev/null
+++ b/docs/user/voice-input.md
@@ -0,0 +1,111 @@
+# Voice input
+
+Dictate into the composer instead of typing. The transcript lands in your draft
+for review, and only sending the message sends anything to an agent. Dictation
+also works in citation comments, diff and file annotations, and pull request
+comments, replies, and review summaries.
+
+On iPhone and Android, voice input transcribes on the device. See
+[Voice input on mobile](./composer.md#voice-input-on-mobile).
+
+## Choose where audio goes
+
+On web and desktop, transcription runs on one of your T3 Code environments.
+Nothing is sent until you choose one: open **Settings → Voice** and set
+**Send microphone audio to** the environment that should transcribe for you.
+Microphone audio recorded on this device is sent to that environment over your
+authenticated connection, transcribed there, and discarded. The choice is saved
+on this device only, so every computer and browser you use asks for it once.
+Set it back to **Off** at any time; a recording in progress is discarded.
+
+If the chosen environment goes offline, voice input waits for it. T3 Code never
+substitutes another environment.
+
+## Set up a model
+
+The first time you dictate, T3 Code offers to download the recommended English
+model to the chosen environment. In **Settings → Voice → Transcription**, select
+the languages you dictate in to compare models that support all of them.
+Accuracy shows how often the model got words or characters wrong in each
+language; speed is the time to transcribe 30 seconds on a reference GPU. Choose
+**All languages** to browse the full catalog. Filtering does not switch your model.
+
+**Advanced** controls GPU or CPU use and when an idle model is unloaded. With hardware set to
+**Auto**, a recording that fails on the GPU is retried once on that same
+environment's CPU; a GPU you picked yourself reports the error instead. Nothing
+is ever retried on another environment. Models marked **Live preview** show
+text while you speak. Other models can process long recordings in chunks at pauses
+while you record; models with built-in long-form support transcribe when you stop.
+
+## Dictate
+
+Press the microphone in the composer, or `mod+shift+d` (`mod` is Command on
+macOS and Ctrl elsewhere). Press again, or the check mark, to finish; `Esc`
+discards the recording and leaves your draft unchanged. Recordings last up to
+five minutes. Text is inserted at the cursor or replaces the selection you had
+when recording started, and the editor is read-only while dictation runs.
+
+Pressing Send while dictating finishes the recording and sends the draft once
+the transcript has been inserted. Cancelling also cancels that queued send. In
+the other editors, Save, Comment, Reply, or Submit review works the same way,
+and the shortcut targets the editor that has focus, or the composer when none
+does.
+
+**Dictation shortcut** in Voice settings picks how the keybinding behaves:
+**Auto** tap-toggles or records while held, **Hold** records only while the keys
+are down, and **Toggle** always needs a second press. Change the chord itself in
+[Keybindings settings](./keybindings.md).
+
+## Improve recognition
+
+- **Language** picks what you speak. **Auto** lets a multilingual model detect
+ it. Models that translate offer **Translate to English**.
+- **Dictionary** holds names and terms the model should get right. Add an alias
+ under a term when transcription keeps producing a particular misspelling.
+ Your dictionary is saved on this device. The name of the project you dictate
+ in is sent along as a hint.
+- **Project dictionary** is vocabulary saved with a project on its environment,
+ so everyone who dictates in that project gets the same spellings. Pick the
+ project at the top of Settings to edit it; with an environment selected, the
+ entries apply to every project there. Project spellings win when they overlap
+ your personal dictionary. Each recording carries up to 100 entries from the
+ two combined.
+- **Silence filtering** is on by default. Your chosen environment skips non-speech
+ audio before transcription. Turn it off to keep those sections. Filtering happens
+ after upload: it does not grant permission to send audio or reduce how much is sent.
+- **Remove filler words** drops hesitation sounds from finished transcripts.
+
+Language and these preferences travel with each recording to the chosen
+environment; nothing else about the project is sent.
+
+## Clean up transcripts with a provider
+
+**Transcript cleanup** is a separate choice and is off by default. When on, the
+finished transcript and up to 1,024 characters each from before, inside, and
+after your selection are sent to the
+text generation provider configured on the draft's environment, which fixes
+punctuation, numbers, and spoken corrections such as "make it 42, sorry, 24".
+Depending on the provider, that can involve an external service. Audio is never
+sent to it. Use the built-in instructions or write your own, press **Skip**
+while cleanup runs to insert the raw transcript, and if cleanup fails the raw
+transcript is inserted.
+
+Cleanup uses the environment's text generation model unless you pick a
+**Cleanup model** for that environment, for example a cheaper or faster one.
+
+If cleanup misses the way you correct yourself, set a **Correction cue**: the
+word or phrase you say to flag a correction, such as "scratch that". Cleanup
+applies the correction only where the context shows one and drops the cue, and
+the model hears the cue as a dictionary hint.
+
+## Test and troubleshoot
+
+In **Settings → Voice**, **Test microphone** beside the microphone selector
+records up to ten seconds and plays it back in your browser;
+the audio stays on this device and is never transcribed, so silent playback
+points at the microphone. **Test model** on the selected model shows a transcript using the
+same environment, language, and dictionary as the composer, but skips transcript
+cleanup even when it is on, so you see what the model heard. Browser recording
+needs HTTPS or localhost and microphone permission; for a remote server over
+plain HTTP, use the desktop app. If the selected microphone is missing, choose
+another or the system default.
diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts
index 8972537c84f..b28c885ce0d 100644
--- a/packages/client-runtime/src/voice-input/controller.test.ts
+++ b/packages/client-runtime/src/voice-input/controller.test.ts
@@ -231,6 +231,21 @@ describe("environment voice input", () => {
});
describe("resolveTranscriptCommit", () => {
+ it.each([
+ ["Hello", "world", "Hello world"],
+ ["Hello", "世界", "Hello世界"],
+ ["你好", "世界", "你好世界"],
+ ])(
+ "preserves script boundaries with an autodetected language: %s + %s",
+ (text, spoken, expected) => {
+ const captured = draft({ text, selection: { start: text.length, end: text.length } });
+ expect(resolveTranscriptCommit(captured, captured, spoken, "auto")).toMatchObject({
+ kind: "commit",
+ text: expected,
+ });
+ },
+ );
+
it("replaces the recorded UTF-16 selection around emoji and composer tokens", () => {
const text = "Fix 🧪 then $review please";
const tokenStart = text.indexOf("$review");
diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts
index 496616b8612..92d743b5e88 100644
--- a/packages/client-runtime/src/voice-input/controller.ts
+++ b/packages/client-runtime/src/voice-input/controller.ts
@@ -11,7 +11,7 @@ const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60;
-const MICROPHONE_PERMISSION_ERROR =
+export const MICROPHONE_PERMISSION_ERROR =
"Microphone access is required for voice input. Allow access in your browser or system settings and try again.";
function isMicrophonePermissionError(error: unknown): boolean {
@@ -131,14 +131,17 @@ export function resolveTranscriptCommit(
const normalizedLocale = locale.replaceAll("_", "-").toLowerCase();
const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-");
let insertion = replacement;
- if (isEmptySelection && usesEnglishSpacing) {
+ const autoSpacing = normalizedLocale === "auto";
+ if (isEmptySelection && (usesEnglishSpacing || autoSpacing)) {
const left = captured.text[captured.selection.start - 1];
const right = captured.text[captured.selection.start];
const leftNeedsBoundary =
+ (!autoSpacing || /^[\p{Script=Latin}\p{N}]/u.test(replacement)) &&
left !== undefined &&
/[A-Za-z0-9.!?,:;)\]}'"]/.test(left) &&
(right === undefined || /\s/.test(right));
const rightNeedsBoundary =
+ (!autoSpacing || /[\p{Script=Latin}\p{N}]$/u.test(replacement)) &&
right !== undefined &&
/[A-Za-z0-9([{'"]/.test(right) &&
(left === undefined || /\s/.test(left));
diff --git a/packages/client-runtime/src/voice-input/environment.ts b/packages/client-runtime/src/voice-input/environment.ts
new file mode 100644
index 00000000000..21166d8b9a0
--- /dev/null
+++ b/packages/client-runtime/src/voice-input/environment.ts
@@ -0,0 +1,208 @@
+import { encodeSpeechPcmRequest } from "@t3tools/shared/speech";
+import * as Effect from "effect/Effect";
+import * as Redacted from "effect/Redacted";
+import { SPEECH_STREAM_PATH } from "@t3tools/contracts";
+import type {
+ SpeechAcceleration,
+ SpeechModelUnloadTimeout,
+ SpeechTranscriptionOptions,
+ SpeechPostProcessingOptions,
+} from "@t3tools/contracts";
+
+import * as RemoteEnvironmentAuthorization from "../authorization/service.ts";
+import type { PreparedConnection } from "../connection/model.ts";
+import * as ManagedRelay from "../relay/managedRelay.ts";
+import {
+ makeEnvironmentHttpApiGroupClient,
+ makeEnvironmentHttpApiUrlBuilder,
+} from "../rpc/http.ts";
+import { executeAuthenticatedEnvironmentHttpRequest } from "../state/environmentHttpAuth.ts";
+
+const INFERENCE_OR_DOWNLOAD_TIMEOUT_MS = 30 * 60_000;
+type EnvironmentApiClient<G extends "voice" | "auth"> = Effect.Success<
+ ReturnType<typeof makeEnvironmentHttpApiGroupClient<G>>
+>;
+
+const request = Effect.fn("clientRuntime.voiceInput.environmentRequest")(function* <
+ A,
+ E,
+ G extends "voice" | "auth",
+>(input: {
+ readonly group: G;
+ readonly prepared: PreparedConnection;
+ readonly method: "GET" | "POST" | "DELETE";
+ readonly timeoutMs?: number;
+ readonly canSend?: () => boolean;
+ readonly path: (urls: ReturnType<typeof makeEnvironmentHttpApiUrlBuilder>[G]) => string;
+ readonly run: (input: {
+ readonly client: EnvironmentApiClient<G>;
+ readonly headers: { readonly authorization?: string; readonly dpop?: string };
+ }) => Effect.Effect<A, E>;
+}) {
+ return yield* executeAuthenticatedEnvironmentHttpRequest({
+ group: input.group,
+ prepared: input.prepared,
+ signer: yield* Effect.serviceOption(ManagedRelay.ManagedRelayDpopSigner),
+ remoteAuthorization: yield* Effect.serviceOption(
+ RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization,
+ ),
+ method: input.method,
+ url: input.path,
+ timeoutMs: input.timeoutMs ?? 30_000,
+ // Audio and credentials never follow a redirect to another host.
+ requestInit: { redirect: "error" },
+ request: (args) =>
+ Effect.suspend(() => (input.canSend?.() === false ? Effect.interrupt : input.run(args))),
+ });
+});
+
+export const getEnvironmentSpeechStatus = (prepared: PreparedConnection) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "GET",
+ path: (urls) => urls.status(),
+ run: ({ client, headers }) => client.status({ headers }),
+ });
+
+export const getEnvironmentSpeechStreamUrl = (prepared: PreparedConnection) => {
+ let endpoint = prepared.httpBaseUrl;
+ return request({
+ group: "auth",
+ prepared,
+ method: "POST",
+ path: (urls) => {
+ endpoint = urls.webSocketTicketFor({ params: { use: "media" } });
+ return endpoint;
+ },
+ run: ({ client, headers }) => client.webSocketTicketFor({ headers, params: { use: "media" } }),
+ }).pipe(
+ Effect.map(({ ticket }) => {
+ const url = new URL(SPEECH_STREAM_PATH, endpoint);
+ url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
+ url.searchParams.set("wsTicket", Redacted.value(ticket));
+ return url.toString();
+ }),
+ );
+};
+
+export const getEnvironmentSpeechModels = (
+ prepared: PreparedConnection,
+ includeBenchmarks = false,
+) => {
+ const query = includeBenchmarks ? { benchmarks: "1" as const } : {};
+ return request({
+ group: "voice",
+ prepared,
+ method: "GET",
+ path: (urls) => urls.models({ query }),
+ run: ({ client, headers }) => client.models({ headers, query }),
+ });
+};
+
+export const downloadEnvironmentSpeechModel = (prepared: PreparedConnection, modelId: string) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "POST",
+ path: (urls) => urls.downloadModel(),
+ timeoutMs: INFERENCE_OR_DOWNLOAD_TIMEOUT_MS,
+ run: ({ client, headers }) => client.downloadModel({ headers, payload: { modelId } }),
+ });
+
+export const selectEnvironmentSpeechModel = (prepared: PreparedConnection, modelId: string) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "POST",
+ path: (urls) => urls.selectModel(),
+ run: ({ client, headers }) => client.selectModel({ headers, payload: { modelId } }),
+ });
+
+export const cancelEnvironmentSpeechModelDownload = (
+ prepared: PreparedConnection,
+ modelId: string,
+) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "POST",
+ path: (urls) => urls.cancelModelDownload(),
+ run: ({ client, headers }) => client.cancelModelDownload({ headers, payload: { modelId } }),
+ });
+
+export const transcribeEnvironmentPcm = (
+ prepared: PreparedConnection,
+ pcm: Uint8Array,
+ options: SpeechTranscriptionOptions,
+ canSend: () => boolean,
+) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "POST",
+ path: (urls) => urls.transcribe(),
+ canSend,
+ timeoutMs: INFERENCE_OR_DOWNLOAD_TIMEOUT_MS,
+ run: ({ client, headers }) =>
+ client.transcribe({
+ headers,
+ payload: encodeSpeechPcmRequest(pcm, options),
+ }),
+ });
+
+/** Sends the transcript and draft context to a provider on `prepared`; a separate opt-in. */
+export const postProcessEnvironmentTranscript = (
+ prepared: PreparedConnection,
+ transcript: string,
+ draft: {
+ readonly text: string;
+ readonly selection: { readonly start: number; readonly end: number };
+ },
+ options: SpeechPostProcessingOptions,
+ canSend: () => boolean,
+) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "POST",
+ path: (urls) => urls.postProcess(),
+ canSend,
+ timeoutMs: INFERENCE_OR_DOWNLOAD_TIMEOUT_MS,
+ run: ({ client, headers }) =>
+ client.postProcess({ headers, payload: { transcript, draft, options } }),
+ });
+
+export const updateEnvironmentSpeechAcceleration = (
+ prepared: PreparedConnection,
+ acceleration: SpeechAcceleration,
+) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "POST",
+ path: (urls) => urls.updateAcceleration(),
+ run: ({ client, headers }) => client.updateAcceleration({ headers, payload: { acceleration } }),
+ });
+
+export const updateEnvironmentSpeechModelUnloadTimeout = (
+ prepared: PreparedConnection,
+ timeout: SpeechModelUnloadTimeout,
+) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "POST",
+ path: (urls) => urls.updateModelUnloadTimeout(),
+ run: ({ client, headers }) =>
+ client.updateModelUnloadTimeout({ headers, payload: { timeout } }),
+ });
+
+export const removeEnvironmentSpeechModel = (prepared: PreparedConnection, modelId: string) =>
+ request({
+ group: "voice",
+ prepared,
+ method: "POST",
+ path: (urls) => urls.removeModel(),
+ run: ({ client, headers }) => client.removeModel({ headers, payload: { modelId } }),
+ });
diff --git a/packages/client-runtime/src/voice-input/index.ts b/packages/client-runtime/src/voice-input/index.ts
index b4dc4b0c172..06a5399d681 100644
--- a/packages/client-runtime/src/voice-input/index.ts
+++ b/packages/client-runtime/src/voice-input/index.ts
@@ -1,6 +1,7 @@
export {
VoiceInputController,
VOICE_RECORDING_LIMIT_SECONDS,
+ MICROPHONE_PERMISSION_ERROR,
voiceInputBlocksSubmission,
type VoiceDraftSnapshot,
type VoiceInputControllerDependencies,
@@ -9,6 +10,20 @@ export {
type VoiceRecorder,
type VoiceRecorderStatus,
} from "./controller.ts";
+export {
+ cancelEnvironmentSpeechModelDownload,
+ downloadEnvironmentSpeechModel,
+ getEnvironmentSpeechModels,
+ getEnvironmentSpeechStatus,
+ getEnvironmentSpeechStreamUrl,
+ postProcessEnvironmentTranscript,
+ removeEnvironmentSpeechModel,
+ selectEnvironmentSpeechModel,
+ transcribeEnvironmentPcm,
+ updateEnvironmentSpeechAcceleration,
+ updateEnvironmentSpeechModelUnloadTimeout,
+} from "./environment.ts";
+export { openSpeechStream } from "./stream.ts";
export {
VoiceTranscriptionError,
throwIfVoiceTranscriptionAborted,
diff --git a/packages/client-runtime/src/voice-input/stream.test.ts b/packages/client-runtime/src/voice-input/stream.test.ts
new file mode 100644
index 00000000000..fc99a618af6
--- /dev/null
+++ b/packages/client-runtime/src/voice-input/stream.test.ts
@@ -0,0 +1,108 @@
+import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS } from "@t3tools/contracts";
+import { expect, it, vi } from "vite-plus/test";
+import { SPEECH_STREAM_MAX_QUEUED_BYTES, type SpeechStreamEvent } from "@t3tools/contracts";
+import { openSpeechStream } from "./stream.ts";
+
+class TestSocket extends EventTarget {
+ sent: (string | Uint8Array)[] = [];
+ closed = false;
+ send(data: string | Uint8Array) {
+ this.sent.push(data);
+ }
+ close() {
+ this.closed = true;
+ this.dispatchEvent(new Event("close"));
+ }
+ receive(event: SpeechStreamEvent) {
+ this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(event) }));
+ }
+}
+
+function connect(bufferedAudio?: Float32Array, canSend = () => true) {
+ const socket = new TestSocket();
+ const abort = new AbortController();
+ const onText = vi.fn();
+ const onError = vi.fn();
+ const stream = openSpeechStream({
+ url: "wss://environment.test/ws/voice",
+ options: DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ signal: abort.signal,
+ canSend,
+ onText,
+ onError,
+ createSocket: () => socket as unknown as WebSocket,
+ });
+ socket.dispatchEvent(new Event("open"));
+ expect(JSON.parse(socket.sent.shift() as string)).toEqual({
+ type: "start",
+ options: DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ });
+ if (bufferedAudio) {
+ stream.feed(bufferedAudio);
+ expect(socket.sent).toHaveLength(0);
+ }
+ socket.receive({ type: "ready" });
+ return { socket, abort, onText, onError, stream };
+}
+
+it("drains ordered PCM before finishing and replaces tentative hypotheses", async () => {
+ const { socket, stream, onText } = connect(new Float32Array([0.1]));
+ stream.feed(new Float32Array([0.2]));
+ const final = stream.finish();
+ expect(socket.sent).toHaveLength(2);
+ socket.receive({ type: "update", revision: 1, text: { committed: "hello ", tentative: "were" } });
+ expect(socket.sent).toHaveLength(2);
+ expect(new Float32Array((socket.sent[1] as Uint8Array).slice().buffer)[0]).toBeCloseTo(0.2);
+ socket.receive({
+ type: "update",
+ revision: 2,
+ text: { committed: "hello ", tentative: "world" },
+ });
+ expect(socket.sent[2]).toBe('{"type":"finish"}');
+ expect(onText.mock.calls.map(([text]) => text)).toEqual([
+ { committed: "hello ", tentative: "were" },
+ { committed: "hello ", tentative: "world" },
+ ]);
+ socket.receive({ type: "finished", text: "hello world" });
+ await expect(final).resolves.toBe("hello world");
+ expect(socket.closed).toBe(true);
+});
+
+it("rejects recordings beyond the duration limit", async () => {
+ const { socket, stream, onError } = connect();
+ stream.feed(new Float32Array(SPEECH_STREAM_MAX_QUEUED_BYTES / 4 + 1));
+ await expect(stream.finish()).rejects.toThrow("limited to five minutes");
+ expect(onError).toHaveBeenCalledOnce();
+ expect(socket.sent).toHaveLength(0);
+ expect(socket.closed).toBe(true);
+});
+
+it.each(["cancel", "revoke"])("does not send queued frames after %s", async (action) => {
+ let allowed = true;
+ const { socket, stream, abort, onText } = connect(undefined, () => allowed);
+ stream.feed(new Float32Array([0.1]));
+ stream.feed(new Float32Array([0.2]));
+ stream.feed(new Float32Array([0.3]));
+ if (action === "cancel") abort.abort();
+ else allowed = false;
+ socket.receive({ type: "update", revision: 1, text: { committed: "late", tentative: "" } });
+ expect(socket.sent).toHaveLength(2);
+ expect(onText).not.toHaveBeenCalled();
+ await expect(stream.finish()).rejects.toThrow(action === "cancel" ? "cancelled" : "revoked");
+});
+
+it("fails on disconnect and times out stalled inference", async () => {
+ vi.useFakeTimers();
+ try {
+ const first = connect();
+ first.socket.close();
+ await expect(first.stream.finish()).rejects.toThrow("connection closed");
+ const second = connect();
+ second.stream.feed(new Float32Array([0.1]));
+ await vi.advanceTimersByTimeAsync(120_000);
+ await expect(second.stream.finish()).rejects.toThrow("stopped responding");
+ expect(second.socket.closed).toBe(true);
+ } finally {
+ vi.useRealTimers();
+ }
+});
diff --git a/packages/client-runtime/src/voice-input/stream.ts b/packages/client-runtime/src/voice-input/stream.ts
new file mode 100644
index 00000000000..ffd4a6991c6
--- /dev/null
+++ b/packages/client-runtime/src/voice-input/stream.ts
@@ -0,0 +1,172 @@
+// @effect-diagnostics globalTimers:off - imperative WebSocket callbacks own and clear their response deadline.
+import {
+ SPEECH_STREAM_MAX_CHUNK_BYTES,
+ SPEECH_STREAM_MAX_QUEUED_BYTES,
+ SpeechStreamEvent,
+ type SpeechStreamText,
+ type SpeechTranscriptionOptions,
+} from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
+
+const decodeEvent = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechStreamEvent));
+
+/** Audio stays bounded while the environment acknowledges a small send window. */
+export function openSpeechStream(input: {
+ readonly url: string;
+ readonly options: SpeechTranscriptionOptions;
+ readonly signal: AbortSignal;
+ readonly canSend: () => boolean;
+ readonly onText: (text: SpeechStreamText) => void;
+ readonly onError: (error: Error) => void;
+ readonly createSocket?: (url: string) => WebSocket;
+}) {
+ input.signal.throwIfAborted();
+ const socket = (input.createSocket ?? ((url) => new WebSocket(url)))(input.url);
+ let resolveFinal!: (text: string) => void;
+ let rejectFinal!: (error: Error) => void;
+ const final = new Promise<string>((resolve, reject) => {
+ resolveFinal = resolve;
+ rejectFinal = reject;
+ });
+ // A capture failure can precede stop(), when nobody is awaiting the final result yet.
+ void final.catch(() => undefined);
+ let queue: Uint8Array<ArrayBuffer>[] = [];
+ let queuedBytes = 0;
+ const inFlight: number[] = [];
+ let inFlightBytes = 0;
+ let finishing = false;
+ let finishSent = false;
+ let closed = false;
+ let isReady = false;
+ let revision = -1;
+ let timer: ReturnType<typeof setTimeout>;
+ const cleanup = () => {
+ clearTimeout(timer);
+ input.signal.removeEventListener("abort", abort);
+ queue = [];
+ queuedBytes = 0;
+ };
+ const fail = (error: Error, notify = true) => {
+ if (closed) return;
+ closed = true;
+ cleanup();
+ rejectFinal(error);
+ socket.close();
+ if (notify) input.onError(error);
+ };
+ const armTimeout = () => {
+ clearTimeout(timer);
+ timer = setTimeout(() => fail(new Error("Live transcription stopped responding.")), 120_000);
+ };
+ const abort = () => fail(new Error("Voice transcription was cancelled."), false);
+ const pump = () => {
+ if (!isReady || closed || finishSent) return;
+ if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
+ while (inFlight.length < 2 && queue.length) {
+ const chunk = queue.shift()!;
+ inFlight.push(chunk.byteLength);
+ inFlightBytes += chunk.byteLength;
+ queuedBytes -= chunk.byteLength;
+ socket.send(chunk);
+ armTimeout();
+ }
+ if (finishing && !queue.length && !inFlight.length) {
+ finishSent = true;
+ socket.send(JSON.stringify({ type: "finish" }));
+ armTimeout();
+ }
+ };
+ socket.addEventListener("open", () => {
+ if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
+ if (!closed) socket.send(JSON.stringify({ type: "start", options: input.options }));
+ });
+ socket.addEventListener("message", (event) => {
+ if (closed) return;
+ if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
+ try {
+ const message = decodeEvent(event.data);
+ switch (message.type) {
+ case "ready":
+ if (isReady) throw new Error("Unexpected speech stream response.");
+ isReady = true;
+ clearTimeout(timer);
+ pump();
+ break;
+ case "update":
+ if (!inFlight.length || message.revision < revision)
+ throw new Error("Unexpected speech stream response.");
+ revision = message.revision;
+ inFlightBytes -= inFlight.shift()!;
+ clearTimeout(timer);
+ if (message.text) input.onText(message.text);
+ pump();
+ if (inFlight.length) armTimeout();
+ break;
+ case "finished":
+ if (!finishSent) throw new Error("Unexpected speech stream response.");
+ closed = true;
+ cleanup();
+ resolveFinal(message.text);
+ socket.close();
+ break;
+ case "error":
+ fail(new Error(message.message));
+ }
+ } catch (error) {
+ fail(error instanceof Error ? error : new Error("Invalid speech stream response."));
+ }
+ });
+ socket.addEventListener("close", () =>
+ fail(new Error("The voice connection closed. Please record again.")),
+ );
+ socket.addEventListener("error", () =>
+ fail(new Error("Could not connect to live transcription.")),
+ );
+ input.signal.addEventListener("abort", abort, { once: true });
+ armTimeout();
+ if (input.signal.aborted) abort();
+ return {
+ feed: (pcm: Float32Array) => {
+ if (closed || finishing) return;
+ const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+ if (queuedBytes + inFlightBytes + bytes.byteLength > SPEECH_STREAM_MAX_QUEUED_BYTES) {
+ fail(new Error("Voice recordings are limited to five minutes."));
+ return;
+ }
+ let offset = 0;
+ while (offset < bytes.length) {
+ const tail = queue.at(-1);
+ const available = tail ? SPEECH_STREAM_MAX_CHUNK_BYTES - tail.byteLength : 0;
+ if (tail && available > 0) {
+ const appended = Math.min(available, bytes.length - offset);
+ const merged = new Uint8Array(tail.byteLength + appended);
+ merged.set(tail);
+ merged.set(bytes.subarray(offset, offset + appended), tail.byteLength);
+ queue[queue.length - 1] = merged;
+ queuedBytes += appended;
+ offset += appended;
+ continue;
+ }
+ const chunk = bytes.slice(offset, offset + SPEECH_STREAM_MAX_CHUNK_BYTES);
+ queue.push(chunk);
+ queuedBytes += chunk.byteLength;
+ offset += chunk.byteLength;
+ }
+ try {
+ pump();
+ } catch {
+ fail(new Error("Could not send microphone audio."));
+ }
+ },
+ finish: () => {
+ finishing = true;
+ try {
+ pump();
+ } catch {
+ fail(new Error("Could not finish live transcription."));
+ }
+ return final;
+ },
+ cancel: abort,
+ };
+}
diff --git a/packages/contracts/src/keybindings.ts b/packages/contracts/src/keybindings.ts
index 714e0a1d486..d17309dcc33 100644
--- a/packages/contracts/src/keybindings.ts
+++ b/packages/contracts/src/keybindings.ts
@@ -90,6 +90,7 @@ export const STATIC_KEYBINDING_COMMANDS = [
"composer.sendAlternate",
"composer.sendBackground",
"composer.sendAndNewThread",
+ "composer.dictation",
"composer.host",
"composer.cycleHost",
"composer.effort",
diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts
index 97dd478327b..a5a27235bbe 100644
--- a/packages/contracts/src/settings.test.ts
+++ b/packages/contracts/src/settings.test.ts
@@ -501,6 +501,10 @@ describe("voice settings", () => {
voiceTranscriptionDestination: { kind: "environment", environmentId: "env-1" },
speechPostProcessingEnabled: true,
};
+ expect(decodeClientSettings({}).speechSilenceFiltering).toBe(true);
+ expect(decodeClientSettingsPatch({ speechSilenceFiltering: false })).toEqual({
+ speechSilenceFiltering: false,
+ });
for (const key of Object.keys(imported)) {
expect(decodeClientSettings(imported)).not.toHaveProperty(key);
expect(decodeClientSettingsPatch(imported)).not.toHaveProperty(key);
diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts
index 212598e3d14..c35193aa6f8 100644
--- a/packages/contracts/src/settings.ts
+++ b/packages/contracts/src/settings.ts
@@ -532,6 +532,7 @@ export const ClientSettingsSchema = Schema.Struct({
speechLanguage: SpeechLanguage.pipe(Schema.withDecodingDefault(Effect.succeed("auto"))),
speechTranslateToEnglish: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),
speechCustomWords: SpeechCustomWords.pipe(Schema.withDecodingDefault(Effect.succeed([]))),
+ speechSilenceFiltering: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
speechRemoveFillerWords: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
speechPostProcessingPrompt: SpeechPostProcessingPromptSettings.pipe(
Schema.withDecodingDefault(
@@ -1892,6 +1893,7 @@ export const ClientSettingsPatch = Schema.Struct({
speechLanguage: Schema.optionalKey(SpeechLanguage),
speechTranslateToEnglish: Schema.optionalKey(Schema.Boolean),
speechCustomWords: Schema.optionalKey(SpeechCustomWords),
+ speechSilenceFiltering: Schema.optionalKey(Schema.Boolean),
speechRemoveFillerWords: Schema.optionalKey(Schema.Boolean),
speechPostProcessingPrompt: Schema.optionalKey(SpeechPostProcessingPromptSettings),
speechCorrectionWord: Schema.optionalKey(SpeechCorrectionCue),
diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts
index 57034071081..5983b6951e0 100644
--- a/packages/contracts/src/speech.ts
+++ b/packages/contracts/src/speech.ts
@@ -1,3 +1,4 @@
+import * as Effect from "effect/Effect";
import * as Schema from "effect/Schema";
export const SpeechModelId = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(160));
@@ -101,6 +102,7 @@ export const EnvironmentSpeechStatus = Schema.Union([
model: Schema.String,
size: Schema.Finite,
supportsStreaming: Schema.Boolean,
+ supportsBufferedStreaming: Schema.optionalKey(Schema.Boolean),
supportsTranslation: Schema.Boolean,
languages: Schema.optionalKey(Schema.Array(Schema.String)),
supportsLanguageDetection: Schema.optionalKey(Schema.Boolean),
@@ -126,12 +128,18 @@ export const SpeechPostProcessingPromptSettings = Schema.Struct({
});
export type SpeechPostProcessingPromptSettings = typeof SpeechPostProcessingPromptSettings.Type;
+/** A word the person says to flag a spoken correction, such as "sorry" or "err". */
+export const SpeechCorrectionCue = Schema.String.check(Schema.isMaxLength(50));
+
/** Recognition preferences that travel with every recording. */
export const SpeechTranscriptionOptions = Schema.Struct({
speechLanguage: SpeechLanguage,
speechTranslateToEnglish: Schema.Boolean,
speechCustomWords: SpeechCustomWords,
speechRemoveFillerWords: Schema.Boolean,
+ /** Preserve this cue for separately consented transcript cleanup. */
+ speechCorrectionWord: Schema.optionalKey(SpeechCorrectionCue),
+ speechSilenceFiltering: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
});
export type SpeechTranscriptionOptions = typeof SpeechTranscriptionOptions.Type;
export const DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS: SpeechTranscriptionOptions = {
@@ -139,12 +147,10 @@ export const DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS: SpeechTranscriptionOptions =
speechTranslateToEnglish: false,
speechCustomWords: [],
speechRemoveFillerWords: true,
+ speechSilenceFiltering: true,
};
export const SPEECH_MAX_OPTIONS_BYTES = 128_000;
-/** A word the person says to flag a spoken correction, such as "sorry" or "err". */
-export const SpeechCorrectionCue = Schema.String.check(Schema.isMaxLength(50));
-
export const SpeechPostProcessingOptions = Schema.Struct({
speechCustomWords: SpeechCustomWords,
speechPostProcessingPrompt: SpeechPostProcessingPromptSettings,
@@ -175,6 +181,7 @@ Preserve the original language, wording, word order, answers, numbers, and negat
export const SPEECH_STREAM_PATH = "/ws/voice";
const SPEECH_SAMPLE_RATE = 16_000;
export const SPEECH_STREAM_MAX_CHUNK_BYTES = SPEECH_SAMPLE_RATE * 4;
+export const SPEECH_STREAM_MAX_QUEUED_BYTES = SPEECH_SAMPLE_RATE * 4 * 5 * 60;
export const SpeechStreamCommand = Schema.Union([
Schema.Struct({ type: Schema.Literal("start"), options: SpeechTranscriptionOptions }),
diff --git a/packages/shared/src/keybindings.ts b/packages/shared/src/keybindings.ts
index 32292c0f9ef..999c00d5317 100644
--- a/packages/shared/src/keybindings.ts
+++ b/packages/shared/src/keybindings.ts
@@ -159,6 +159,7 @@ export const DEFAULT_KEYBINDINGS: ReadonlyArray<KeybindingRule> = [
{ key: "mod+t", command: "rightPanel.new", when: "rightPanelOpen && !terminalFocus" },
{ key: "mod+d", command: "terminal.split", when: "terminalFocus" },
{ key: "mod+shift+d", command: "terminal.splitVertical", when: "terminalFocus" },
+ { key: "mod+shift+d", command: "composer.dictation", when: "!terminalFocus" },
{ key: "mod+n", command: "terminal.new", when: "terminalFocus" },
{ key: "mod+w", command: "terminal.close", when: "terminalFocus" },
{ key: "mod+w", command: "rightPanel.close", when: "!terminalFocus" },
diff --git a/packages/shared/src/speech.test.ts b/packages/shared/src/speech.test.ts
index 948ec2c1ff4..8ef4e7fd20a 100644
--- a/packages/shared/src/speech.test.ts
+++ b/packages/shared/src/speech.test.ts
@@ -17,6 +17,26 @@ describe("speech PCM request framing", () => {
expect(decodeSpeechPcmRequest(pool.subarray(3))).toEqual({ pcm, options });
});
+ it("accepts recordings from clients that predate silence filtering", () => {
+ const metadata = new TextEncoder().encode(
+ JSON.stringify({
+ speechLanguage: "fr",
+ speechTranslateToEnglish: false,
+ speechCustomWords: [],
+ speechRemoveFillerWords: false,
+ }),
+ );
+ const body = new Uint8Array(4 + metadata.length + 4);
+ new DataView(body.buffer).setUint32(0, metadata.length, true);
+ body.set(metadata, 4);
+ expect(decodeSpeechPcmRequest(body).options).toEqual({
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
+ speechLanguage: "fr",
+ speechRemoveFillerWords: false,
+ speechSilenceFiltering: true,
+ });
+ });
+
it("rejects truncated frames, oversized option lengths and missing audio", () => {
expect(() => decodeSpeechPcmRequest(new Uint8Array(3))).toThrow();
const oversized = new Uint8Array(5);