Skip to content
TypeScript

fix(web): relay-connected microphone destinations authorize speech requests (#31778)

Julius Marminged5fc29fe5cb6
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
index ab1c0187c28..e62e7180fe0 100644
--- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
@@ -1,4 +1,5 @@
-import { PrimaryConnectionTarget } from "@t3tools/client-runtime/connection";
+import { PrimaryConnectionTarget, RelayConnectionTarget } from "@t3tools/client-runtime/connection";
+import type * as Effect from "effect/Effect";
import {
EnvironmentId,
type EnvironmentSpeechModel,
@@ -19,6 +20,53 @@ import { SpeechLanguageSelect } from "./SpeechLanguageSelect";
import { VoiceSettingsPanel } from "./VoiceSettingsPanel";
vi.mock(import("../../speech/useEnvironmentSpeechInput"), { spy: true });
+// The real connection runtime authorizes relay requests with a signed-in cloud session.
+vi.mock("../../connection/runtime", async () => {
+ const [{ ManagedRelay }, rpc, { Atom }, Context, Effect, Layer] = await Promise.all([
+ import("@t3tools/client-runtime/relay"),
+ import("@t3tools/client-runtime/rpc"),
+ import("effect/reactivity"),
+ import("effect/Context"),
+ import("effect/Effect"),
+ import("effect/Layer"),
+ ]);
+ type AuthorizedHttp =
+ import("@t3tools/client-runtime/authorization").AuthorizedRemoteHttpEnvironment;
+ // Services resolve by key, and client-runtime does not export this one.
+ class RemoteEnvironmentAuthorization extends Context.Service<
+ RemoteEnvironmentAuthorization,
+ {
+ readonly authorizeDpopHttp: (input: {
+ readonly expectedEnvironmentId: AuthorizedHttp["environmentId"];
+ }) => Effect.Effect<AuthorizedHttp>;
+ }
+ // @effect-diagnostics-next-line deterministicKeys:off
+ >()("@t3tools/client-runtime/authorization/service/RemoteEnvironmentAuthorization") {}
+ return {
+ connectionAtomRuntime: Atom.runtime(
+ Layer.mergeAll(
+ rpc.layerRemoteHttpClient((input, init) => globalThis.fetch(input, init)),
+ Layer.succeed(ManagedRelay.ManagedRelayDpopSigner, {
+ thumbprint: Effect.succeed("thumbprint"),
+ createProof: () => Effect.succeed("proof"),
+ }),
+ Layer.succeed(RemoteEnvironmentAuthorization, {
+ authorizeDpopHttp: ({ expectedEnvironmentId }) =>
+ Effect.succeed({
+ environmentId: expectedEnvironmentId,
+ label: "MacBook Pro",
+ httpBaseUrl: window.location.origin,
+ httpAuthorization: {
+ _tag: "Dpop",
+ accessToken: "relay-access-token",
+ expiresAtEpochMs: 4_102_444_800_000,
+ },
+ }),
+ }),
+ ),
+ ),
+ };
+});
it("preserves the model test transcript when its model is filtered out", async () => {
const target = new PrimaryConnectionTarget({
@@ -157,6 +205,81 @@ it("preserves the model test transcript when its model is filtered out", async (
);
});
+it("loads the models of a relay-connected destination", async () => {
+ const target = new RelayConnectionTarget({
+ environmentId: EnvironmentId.make("voice-relay"),
+ label: "MacBook Pro",
+ });
+ const prepared = {
+ environmentId: target.environmentId,
+ label: target.label,
+ httpBaseUrl: window.location.origin,
+ socketUrl: "ws://localhost",
+ httpAuthorization: {
+ _tag: "Dpop" as const,
+ accessToken: "stale-access-token",
+ expiresAtEpochMs: 4_102_444_800_000,
+ },
+ target,
+ };
+ const model: EnvironmentSpeechModel = {
+ id: "tiny",
+ name: "Tiny",
+ description: "",
+ size: 100,
+ languages: ["en"],
+ accuracy: 1,
+ speed: 1,
+ recommended: false,
+ supportsStreaming: false,
+ supportsTranslation: false,
+ supportsLanguageDetection: false,
+ active: false,
+ state: "downloadable",
+ };
+ const status = {
+ supported: true,
+ state: "missing-model",
+ modelId: model.id,
+ model: model.name,
+ size: model.size,
+ supportsStreaming: false,
+ supportsTranslation: false,
+ acceleration: "cpu",
+ modelUnloadTimeout: "never",
+ gpuDevices: [],
+ } satisfies EnvironmentSpeechStatus;
+ const authorizations: Array<string | null> = [];
+ const fetch = globalThis.fetch;
+ using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
+ const url = new URL(input instanceof Request ? input.url : input, window.location.href);
+ if (url.pathname === "/api/voice/status") {
+ authorizations.push(new Headers(init?.headers).get("authorization"));
+ return Promise.resolve(Response.json(status));
+ }
+ if (url.pathname === "/api/voice/models")
+ return Promise.resolve(Response.json({ models: [model] }));
+ return fetch(input, init);
+ });
+ using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");
+ // Restored even when an assertion fails, unlike `withImplementation`.
+ using _destination = vi.mocked(useVoiceTranscriptionDestination).mockImplementation(() => ({
+ environmentId: target.environmentId,
+ label: target.label,
+ grantId: "test",
+ prepared,
+ }));
+ await using screen = await render(
+ <SettingsScopeProvider search={{}} onChange={() => {}}>
+ <VoiceSettingsPanel />
+ </SettingsScopeProvider>,
+ );
+ await vi.waitFor(() => expect(authorizations.length + notify.mock.calls.length).not.toBe(0));
+ expect(notify).not.toHaveBeenCalled();
+ expect(authorizations).toEqual(["DPoP relay-access-token"]);
+ await expect.element(screen.getByText("Tiny", { exact: true })).toBeVisible();
+});
+
it("recovers from a failed download progress poll without notifying, but reports a failed initial load", async () => {
const target = new PrimaryConnectionTarget({
environmentId: EnvironmentId.make("voice-progress"),
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx
index b0fd5d2fff5..fc351783126 100644
--- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx
+++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx
@@ -45,7 +45,7 @@ import {
useCleanupGrant,
} from "../../speech/voiceConsent";
import { updateClientSettings, useClientSettings } from "../../hooks/useSettings";
-import { runtime } from "../../lib/runtime";
+import { runEnvironmentSpeech } from "../../speech/runEnvironmentSpeech";
import {
getCustomModelOptionsByInstance,
resolveAppModelSelectionState,
@@ -542,8 +542,8 @@ function useVoiceModels(prepared: PreparedConnection | null) {
const revision = ++refreshRevision.current;
try {
const [nextStatus, nextModels] = await Promise.all([
- runtime.runPromise(getEnvironmentSpeechStatus(connection)),
- runtime.runPromise(getEnvironmentSpeechModels(connection, mode === "foreground")),
+ runEnvironmentSpeech(getEnvironmentSpeechStatus(connection)),
+ runEnvironmentSpeech(getEnvironmentSpeechModels(connection, mode === "foreground")),
]);
if (current !== epoch.current || revision !== refreshRevision.current) return;
setStatus(nextStatus);
@@ -649,7 +649,7 @@ function AdvancedSpeechSettings({
onValueChange={(value) => {
if (value)
void run("Could not change transcription hardware", () =>
- runtime.runPromise(
+ runEnvironmentSpeech(
updateEnvironmentSpeechAcceleration(prepared, value as SpeechAcceleration),
),
);
@@ -688,7 +688,7 @@ function AdvancedSpeechSettings({
onValueChange={(value) => {
if (value)
void run("Could not change when the model unloads", () =>
- runtime.runPromise(
+ runEnvironmentSpeech(
updateEnvironmentSpeechModelUnloadTimeout(
prepared,
value as SpeechModelUnloadTimeout,
@@ -896,22 +896,22 @@ function SpeechModelBrowser(props: {
void run(
"Could not download the model",
() =>
- runtime.runPromise(downloadEnvironmentSpeechModel(prepared, model.id)),
+ runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, model.id)),
model.id,
),
onSelect: () =>
void run("Could not select the model", () =>
- runtime.runPromise(selectEnvironmentSpeechModel(prepared, model.id)),
+ runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, model.id)),
),
onCancel: () =>
void run("Could not cancel the download", () =>
- runtime.runPromise(
+ runEnvironmentSpeech(
cancelEnvironmentSpeechModelDownload(prepared, model.id),
),
),
onDelete: () =>
void run("Could not delete the model", () =>
- runtime.runPromise(removeEnvironmentSpeechModel(prepared, model.id)),
+ runEnvironmentSpeech(removeEnvironmentSpeechModel(prepared, model.id)),
),
};
return model.active ? (
diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts
index 1d04f96bba0..5690773a610 100644
--- a/apps/web/src/speech/browserVoiceInput.ts
+++ b/apps/web/src/speech/browserVoiceInput.ts
@@ -11,7 +11,7 @@ import {
import type { PreparedConnection } from "@t3tools/client-runtime/connection";
import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts";
-import { runtime } from "../lib/runtime";
+import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
import { createBrowserPcmCapture } from "./browserPcmCapture";
export function createBrowserVoiceInputPlatform(input: {
@@ -26,7 +26,7 @@ export function createBrowserVoiceInputPlatform(input: {
}) {
return createBrowserPcmCapture(input, async (options) => {
const recordingOptions = input.getTranscriptionOptions();
- const status = await runtime.runPromise(getEnvironmentSpeechStatus(input.prepared), options);
+ const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
throwIfVoiceTranscriptionAborted(options.signal);
if (!status.supported)
throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
@@ -52,7 +52,7 @@ export function createBrowserVoiceInputPlatform(input: {
try {
const response = await fetch(uri, { signal: transcriptionSignal });
const pcm = new Uint8Array(await response.arrayBuffer());
- const result = await runtime.runPromise(
+ const result = await runEnvironmentSpeech(
transcribeEnvironmentPcm(input.prepared, pcm, recordingOptions, input.hasAudioConsent),
{
signal: transcriptionSignal,
@@ -72,7 +72,7 @@ export function createBrowserVoiceInputPlatform(input: {
if (!status.supportsStreaming && !status.supportsBufferedStreaming) {
return { locale, transcribe: transcribeRecording };
}
- const url = await runtime.runPromise(getEnvironmentSpeechStreamUrl(input.prepared), options);
+ const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options);
return {
locale,
transcribe: transcribeRecording,
diff --git a/apps/web/src/speech/runEnvironmentSpeech.ts b/apps/web/src/speech/runEnvironmentSpeech.ts
new file mode 100644
index 00000000000..bc6d1db747c
--- /dev/null
+++ b/apps/web/src/speech/runEnvironmentSpeech.ts
@@ -0,0 +1,19 @@
+import * as Effect from "effect/Effect";
+import type { HttpClient } from "effect/http";
+import { AtomRegistry } from "effect/reactivity";
+
+import { connectionAtomRuntime } from "../connection/runtime";
+import { appAtomRegistry } from "../rpc/atomRegistry";
+
+// Relay (DPoP) environments authorize through services only the connection runtime provides.
+export function runEnvironmentSpeech<A, E>(
+ effect: Effect.Effect<A, E, HttpClient.HttpClient>,
+ options?: Effect.RunOptions,
+): Promise<A> {
+ return Effect.runPromise(
+ AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(
+ Effect.flatMap((context) => Effect.provide(effect, context)),
+ ),
+ options,
+ );
+}
diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
index a41f359f68e..04534c3e0cf 100644
--- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
@@ -88,7 +88,7 @@ vi.mock("../hooks/useSettings", () => ({
),
}),
}));
-vi.mock("../lib/runtime", () => ({ runtime: { runPromise: Effect.runPromise } }));
+vi.mock("./runEnvironmentSpeech", () => ({ runEnvironmentSpeech: Effect.runPromise }));
vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } }));
vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
...(await importOriginal<typeof import("@t3tools/client-runtime/voice-input")>()),
diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts
index 644edb303ae..0d1853a5dcc 100644
--- a/apps/web/src/speech/useEnvironmentSpeechInput.ts
+++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts
@@ -33,10 +33,10 @@ import {
useClientSettingsHydrated,
useEnvironmentSettings,
} from "../hooks/useSettings";
-import { runtime } from "../lib/runtime";
import { useEnvironment } from "../state/environments";
import { useEnvironmentScope, usePreparedConnection } from "../state/session";
import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
+import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
import { bindAudioGrant, readCleanupGrant, useAudioGrant } from "./voiceConsent";
const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
@@ -54,7 +54,7 @@ const statusProbes = new WeakMap<PreparedConnection, Promise<EnvironmentSpeechSt
function probeSpeechStatus(prepared: PreparedConnection, refresh = false) {
const cached = refresh ? undefined : statusProbes.get(prepared);
if (cached) return cached;
- const probe = runtime.runPromise(getEnvironmentSpeechStatus(prepared));
+ const probe = runEnvironmentSpeech(getEnvironmentSpeechStatus(prepared));
statusProbes.set(prepared, probe);
probe.catch(() => {
if (statusProbes.get(prepared) === probe) statusProbes.delete(prepared);
@@ -296,18 +296,16 @@ export function useEnvironmentSpeechInput(input: HookInput) {
return Promise.reject(
new Error("The draft's environment is unavailable for transcript cleanup."),
);
- return runtime
- .runPromise(
- postProcessEnvironmentTranscript(
- target,
- transcript,
- cleanupContext(options.draft),
- preferences.cleanup,
- () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
- ),
- options,
- )
- .then((result) => result.text);
+ return runEnvironmentSpeech(
+ postProcessEnvironmentTranscript(
+ target,
+ transcript,
+ cleanupContext(options.draft),
+ preferences.cleanup,
+ () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
+ ),
+ options,
+ ).then((result) => result.text);
};
controller = new VoiceInputController({
retryFailedRecording: true,
@@ -374,8 +372,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
const timer = window.setInterval(() => {
if (refreshing) return;
refreshing = true;
- void runtime
- .runPromise(getEnvironmentSpeechModels(prepared))
+ void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared))
.then((result) => {
if (disposed) return;
const model = result.models.find((candidate) => candidate.id === modelId) ?? null;
@@ -471,7 +468,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
const request = ++startRequestRef.current;
setSetup((current) => ({ ...current, downloading: true, error: null }));
try {
- await runtime.runPromise(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));
+ await runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));
if (request !== startRequestRef.current) return;
const next = await probeSpeechStatus(prepared, true);
if (request !== startRequestRef.current) return;
@@ -500,7 +497,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
startRequestRef.current += 1;
setSetup((current) => ({ ...current, open: false, downloading: false }));
try {
- await runtime.runPromise(
+ await runEnvironmentSpeech(
cancelEnvironmentSpeechModelDownload(prepared, currentStatus.modelId),
);
} catch (error) {