TypeScript
fix(web): relay-connected microphone destinations authorize speech requests (#31778)
Julius Marminged5fc29fe5cb6
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsxindex ab1c0187c28..e62e7180fe0 100644--- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx+++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx@@ -1,4 +1,5 @@-import { PrimaryConnectionTarget } from "@t3tools/client-runtime/connection";+import { PrimaryConnectionTarget, RelayConnectionTarget } from "@t3tools/client-runtime/connection";+import type * as Effect from "effect/Effect";import {EnvironmentId,type EnvironmentSpeechModel,@@ -19,6 +20,53 @@ import { SpeechLanguageSelect } from "./SpeechLanguageSelect";import { VoiceSettingsPanel } from "./VoiceSettingsPanel";vi.mock(import("../../speech/useEnvironmentSpeechInput"), { spy: true });+// The real connection runtime authorizes relay requests with a signed-in cloud session.+vi.mock("../../connection/runtime", async () => {+ const [{ ManagedRelay }, rpc, { Atom }, Context, Effect, Layer] = await Promise.all([+ import("@t3tools/client-runtime/relay"),+ import("@t3tools/client-runtime/rpc"),+ import("effect/reactivity"),+ import("effect/Context"),+ import("effect/Effect"),+ import("effect/Layer"),+ ]);+ type AuthorizedHttp =+ import("@t3tools/client-runtime/authorization").AuthorizedRemoteHttpEnvironment;+ // Services resolve by key, and client-runtime does not export this one.+ class RemoteEnvironmentAuthorization extends Context.Service<+ RemoteEnvironmentAuthorization,+ {+ readonly authorizeDpopHttp: (input: {+ readonly expectedEnvironmentId: AuthorizedHttp["environmentId"];+ }) => Effect.Effect<AuthorizedHttp>;+ }+ // @effect-diagnostics-next-line deterministicKeys:off+ >()("@t3tools/client-runtime/authorization/service/RemoteEnvironmentAuthorization") {}+ return {+ connectionAtomRuntime: Atom.runtime(+ Layer.mergeAll(+ rpc.layerRemoteHttpClient((input, init) => globalThis.fetch(input, init)),+ Layer.succeed(ManagedRelay.ManagedRelayDpopSigner, {+ thumbprint: Effect.succeed("thumbprint"),+ createProof: () => Effect.succeed("proof"),+ }),+ Layer.succeed(RemoteEnvironmentAuthorization, {+ authorizeDpopHttp: ({ expectedEnvironmentId }) =>+ Effect.succeed({+ environmentId: expectedEnvironmentId,+ label: "MacBook Pro",+ httpBaseUrl: window.location.origin,+ httpAuthorization: {+ _tag: "Dpop",+ accessToken: "relay-access-token",+ expiresAtEpochMs: 4_102_444_800_000,+ },+ }),+ }),+ ),+ ),+ };+});it("preserves the model test transcript when its model is filtered out", async () => {const target = new PrimaryConnectionTarget({@@ -157,6 +205,81 @@ it("preserves the model test transcript when its model is filtered out", async ();});+it("loads the models of a relay-connected destination", async () => {+ const target = new RelayConnectionTarget({+ environmentId: EnvironmentId.make("voice-relay"),+ label: "MacBook Pro",+ });+ const prepared = {+ environmentId: target.environmentId,+ label: target.label,+ httpBaseUrl: window.location.origin,+ socketUrl: "ws://localhost",+ httpAuthorization: {+ _tag: "Dpop" as const,+ accessToken: "stale-access-token",+ expiresAtEpochMs: 4_102_444_800_000,+ },+ target,+ };+ const model: EnvironmentSpeechModel = {+ id: "tiny",+ name: "Tiny",+ description: "",+ size: 100,+ languages: ["en"],+ accuracy: 1,+ speed: 1,+ recommended: false,+ supportsStreaming: false,+ supportsTranslation: false,+ supportsLanguageDetection: false,+ active: false,+ state: "downloadable",+ };+ const status = {+ supported: true,+ state: "missing-model",+ modelId: model.id,+ model: model.name,+ size: model.size,+ supportsStreaming: false,+ supportsTranslation: false,+ acceleration: "cpu",+ modelUnloadTimeout: "never",+ gpuDevices: [],+ } satisfies EnvironmentSpeechStatus;+ const authorizations: Array<string | null> = [];+ const fetch = globalThis.fetch;+ using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {+ const url = new URL(input instanceof Request ? input.url : input, window.location.href);+ if (url.pathname === "/api/voice/status") {+ authorizations.push(new Headers(init?.headers).get("authorization"));+ return Promise.resolve(Response.json(status));+ }+ if (url.pathname === "/api/voice/models")+ return Promise.resolve(Response.json({ models: [model] }));+ return fetch(input, init);+ });+ using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");+ // Restored even when an assertion fails, unlike `withImplementation`.+ using _destination = vi.mocked(useVoiceTranscriptionDestination).mockImplementation(() => ({+ environmentId: target.environmentId,+ label: target.label,+ grantId: "test",+ prepared,+ }));+ await using screen = await render(+ <SettingsScopeProvider search={{}} onChange={() => {}}>+ <VoiceSettingsPanel />+ </SettingsScopeProvider>,+ );+ await vi.waitFor(() => expect(authorizations.length + notify.mock.calls.length).not.toBe(0));+ expect(notify).not.toHaveBeenCalled();+ expect(authorizations).toEqual(["DPoP relay-access-token"]);+ await expect.element(screen.getByText("Tiny", { exact: true })).toBeVisible();+});+it("recovers from a failed download progress poll without notifying, but reports a failed initial load", async () => {const target = new PrimaryConnectionTarget({environmentId: EnvironmentId.make("voice-progress"),diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsxindex b0fd5d2fff5..fc351783126 100644--- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx+++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx@@ -45,7 +45,7 @@ import {useCleanupGrant,} from "../../speech/voiceConsent";import { updateClientSettings, useClientSettings } from "../../hooks/useSettings";-import { runtime } from "../../lib/runtime";+import { runEnvironmentSpeech } from "../../speech/runEnvironmentSpeech";import {getCustomModelOptionsByInstance,resolveAppModelSelectionState,@@ -542,8 +542,8 @@ function useVoiceModels(prepared: PreparedConnection | null) {const revision = ++refreshRevision.current;try {const [nextStatus, nextModels] = await Promise.all([- runtime.runPromise(getEnvironmentSpeechStatus(connection)),- runtime.runPromise(getEnvironmentSpeechModels(connection, mode === "foreground")),+ runEnvironmentSpeech(getEnvironmentSpeechStatus(connection)),+ runEnvironmentSpeech(getEnvironmentSpeechModels(connection, mode === "foreground")),]);if (current !== epoch.current || revision !== refreshRevision.current) return;setStatus(nextStatus);@@ -649,7 +649,7 @@ function AdvancedSpeechSettings({onValueChange={(value) => {if (value)void run("Could not change transcription hardware", () =>- runtime.runPromise(+ runEnvironmentSpeech(updateEnvironmentSpeechAcceleration(prepared, value as SpeechAcceleration),),);@@ -688,7 +688,7 @@ function AdvancedSpeechSettings({onValueChange={(value) => {if (value)void run("Could not change when the model unloads", () =>- runtime.runPromise(+ runEnvironmentSpeech(updateEnvironmentSpeechModelUnloadTimeout(prepared,value as SpeechModelUnloadTimeout,@@ -896,22 +896,22 @@ function SpeechModelBrowser(props: {void run("Could not download the model",() =>- runtime.runPromise(downloadEnvironmentSpeechModel(prepared, model.id)),+ runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, model.id)),model.id,),onSelect: () =>void run("Could not select the model", () =>- runtime.runPromise(selectEnvironmentSpeechModel(prepared, model.id)),+ runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, model.id)),),onCancel: () =>void run("Could not cancel the download", () =>- runtime.runPromise(+ runEnvironmentSpeech(cancelEnvironmentSpeechModelDownload(prepared, model.id),),),onDelete: () =>void run("Could not delete the model", () =>- runtime.runPromise(removeEnvironmentSpeechModel(prepared, model.id)),+ runEnvironmentSpeech(removeEnvironmentSpeechModel(prepared, model.id)),),};return model.active ? (diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.tsindex 1d04f96bba0..5690773a610 100644--- a/apps/web/src/speech/browserVoiceInput.ts+++ b/apps/web/src/speech/browserVoiceInput.ts@@ -11,7 +11,7 @@ import {import type { PreparedConnection } from "@t3tools/client-runtime/connection";import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts";-import { runtime } from "../lib/runtime";+import { runEnvironmentSpeech } from "./runEnvironmentSpeech";import { createBrowserPcmCapture } from "./browserPcmCapture";export function createBrowserVoiceInputPlatform(input: {@@ -26,7 +26,7 @@ export function createBrowserVoiceInputPlatform(input: {}) {return createBrowserPcmCapture(input, async (options) => {const recordingOptions = input.getTranscriptionOptions();- const status = await runtime.runPromise(getEnvironmentSpeechStatus(input.prepared), options);+ const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);throwIfVoiceTranscriptionAborted(options.signal);if (!status.supported)throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });@@ -52,7 +52,7 @@ export function createBrowserVoiceInputPlatform(input: {try {const response = await fetch(uri, { signal: transcriptionSignal });const pcm = new Uint8Array(await response.arrayBuffer());- const result = await runtime.runPromise(+ const result = await runEnvironmentSpeech(transcribeEnvironmentPcm(input.prepared, pcm, recordingOptions, input.hasAudioConsent),{signal: transcriptionSignal,@@ -72,7 +72,7 @@ export function createBrowserVoiceInputPlatform(input: {if (!status.supportsStreaming && !status.supportsBufferedStreaming) {return { locale, transcribe: transcribeRecording };}- const url = await runtime.runPromise(getEnvironmentSpeechStreamUrl(input.prepared), options);+ const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options);return {locale,transcribe: transcribeRecording,diff --git a/apps/web/src/speech/runEnvironmentSpeech.ts b/apps/web/src/speech/runEnvironmentSpeech.tsnew file mode 100644index 00000000000..bc6d1db747c--- /dev/null+++ b/apps/web/src/speech/runEnvironmentSpeech.ts@@ -0,0 +1,19 @@+import * as Effect from "effect/Effect";+import type { HttpClient } from "effect/http";+import { AtomRegistry } from "effect/reactivity";++import { connectionAtomRuntime } from "../connection/runtime";+import { appAtomRegistry } from "../rpc/atomRegistry";++// Relay (DPoP) environments authorize through services only the connection runtime provides.+export function runEnvironmentSpeech<A, E>(+ effect: Effect.Effect<A, E, HttpClient.HttpClient>,+ options?: Effect.RunOptions,+): Promise<A> {+ return Effect.runPromise(+ AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(+ Effect.flatMap((context) => Effect.provide(effect, context)),+ ),+ options,+ );+}diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsxindex a41f359f68e..04534c3e0cf 100644--- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx+++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx@@ -88,7 +88,7 @@ vi.mock("../hooks/useSettings", () => ({),}),}));-vi.mock("../lib/runtime", () => ({ runtime: { runPromise: Effect.runPromise } }));+vi.mock("./runEnvironmentSpeech", () => ({ runEnvironmentSpeech: Effect.runPromise }));vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } }));vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({...(await importOriginal<typeof import("@t3tools/client-runtime/voice-input")>()),diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.tsindex 644edb303ae..0d1853a5dcc 100644--- a/apps/web/src/speech/useEnvironmentSpeechInput.ts+++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts@@ -33,10 +33,10 @@ import {useClientSettingsHydrated,useEnvironmentSettings,} from "../hooks/useSettings";-import { runtime } from "../lib/runtime";import { useEnvironment } from "../state/environments";import { useEnvironmentScope, usePreparedConnection } from "../state/session";import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";+import { runEnvironmentSpeech } from "./runEnvironmentSpeech";import { bindAudioGrant, readCleanupGrant, useAudioGrant } from "./voiceConsent";const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };@@ -54,7 +54,7 @@ const statusProbes = new WeakMap<PreparedConnection, Promise<EnvironmentSpeechStfunction probeSpeechStatus(prepared: PreparedConnection, refresh = false) {const cached = refresh ? undefined : statusProbes.get(prepared);if (cached) return cached;- const probe = runtime.runPromise(getEnvironmentSpeechStatus(prepared));+ const probe = runEnvironmentSpeech(getEnvironmentSpeechStatus(prepared));statusProbes.set(prepared, probe);probe.catch(() => {if (statusProbes.get(prepared) === probe) statusProbes.delete(prepared);@@ -296,18 +296,16 @@ export function useEnvironmentSpeechInput(input: HookInput) {return Promise.reject(new Error("The draft's environment is unavailable for transcript cleanup."),);- return runtime- .runPromise(- postProcessEnvironmentTranscript(- target,- transcript,- cleanupContext(options.draft),- preferences.cleanup,- () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),- ),- options,- )- .then((result) => result.text);+ return runEnvironmentSpeech(+ postProcessEnvironmentTranscript(+ target,+ transcript,+ cleanupContext(options.draft),+ preferences.cleanup,+ () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),+ ),+ options,+ ).then((result) => result.text);};controller = new VoiceInputController({retryFailedRecording: true,@@ -374,8 +372,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {const timer = window.setInterval(() => {if (refreshing) return;refreshing = true;- void runtime- .runPromise(getEnvironmentSpeechModels(prepared))+ void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared)).then((result) => {if (disposed) return;const model = result.models.find((candidate) => candidate.id === modelId) ?? null;@@ -471,7 +468,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {const request = ++startRequestRef.current;setSetup((current) => ({ ...current, downloading: true, error: null }));try {- await runtime.runPromise(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));+ await runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));if (request !== startRequestRef.current) return;const next = await probeSpeechStatus(prepared, true);if (request !== startRequestRef.current) return;@@ -500,7 +497,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {startRequestRef.current += 1;setSetup((current) => ({ ...current, open: false, downloading: false }));try {- await runtime.runPromise(+ await runEnvironmentSpeech(cancelEnvironmentSpeechModelDownload(prepared, currentStatus.modelId),);} catch (error) {