Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

feat(voice): add local desktop bot calls (#11)

Leof6a582a3b98c
diff --git a/apps/desktop/scripts/electron-launcher.mjs b/apps/desktop/scripts/electron-launcher.mjs
index b0ec74d48..f5e14f167 100644
--- a/apps/desktop/scripts/electron-launcher.mjs
+++ b/apps/desktop/scripts/electron-launcher.mjs
@@ -228,6 +228,11 @@ function patchMainBundleInfoPlist(appBundlePath, iconPath, executableName) {
setPlistString(infoPlistPath, "CFBundleIdentifier", APP_BUNDLE_ID);
setPlistString(infoPlistPath, "CFBundleExecutable", executableName);
setPlistString(infoPlistPath, "CFBundleIconFile", "icon.icns");
+ setPlistString(
+ infoPlistPath,
+ "NSMicrophoneUsageDescription",
+ "Akeru Bot uses the microphone for calls with your bots.",
+ );
setPlistJson(infoPlistPath, "CFBundleURLTypes", [
{
CFBundleURLName: APP_BUNDLE_ID,
diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts
index 03fbb3a6f..464ac3bf5 100644
--- a/apps/server/src/auth/RpcAuthorization.ts
+++ b/apps/server/src/auth/RpcAuthorization.ts
@@ -45,6 +45,9 @@ export const RPC_REQUIRED_SCOPES = {
[WS_METHODS.subscriptionAuthComplete]: AuthOrchestrationOperateScope,
[WS_METHODS.subscriptionAuthCancel]: AuthOrchestrationOperateScope,
[WS_METHODS.subscriptionAuthLogout]: AuthOrchestrationOperateScope,
+ [WS_METHODS.voiceCallGet]: AuthOrchestrationReadScope,
+ [WS_METHODS.voiceCallStart]: AuthOrchestrationOperateScope,
+ [WS_METHODS.voiceCallHangup]: AuthOrchestrationOperateScope,
[WS_METHODS.serverDiscoverSourceControl]: AuthOrchestrationReadScope,
[WS_METHODS.serverGetTraceDiagnostics]: AuthOrchestrationReadScope,
[WS_METHODS.serverGetProcessDiagnostics]: AuthOrchestrationReadScope,
diff --git a/apps/server/src/orchestration/Layers/BotPersistence.test.ts b/apps/server/src/orchestration/Layers/BotPersistence.test.ts
index 475d5e621..e3ca1c177 100644
--- a/apps/server/src/orchestration/Layers/BotPersistence.test.ts
+++ b/apps/server/src/orchestration/Layers/BotPersistence.test.ts
@@ -94,6 +94,7 @@ it.layer(TestLayer)("bot persistence", (it) => {
sandbox: null,
runtimeMode: "approval-required",
usageCap: { unit: "tokens", limit: 50_000 },
+ voiceEnabled: true,
});
yield* engine.dispatch({
type: "bot.archive",
@@ -131,6 +132,7 @@ it.layer(TestLayer)("bot persistence", (it) => {
sandbox: null,
runtimeMode: "approval-required",
usageCap: { unit: "tokens", limit: 50_000 },
+ voiceEnabled: true,
groupId,
archivedAt: null,
createdAt,
@@ -167,6 +169,7 @@ it.layer(TestLayer)("bot persistence", (it) => {
assert.equal(rebuilt.bots.length, 1);
assert.equal(rebuilt.bots[0]?.id, botId);
assert.equal(rebuilt.bots[0]?.name, "Pathfinder");
+ assert.equal(rebuilt.bots[0]?.voiceEnabled, true);
assert.equal(rebuilt.bots[0]?.groupId, null);
assert.equal(rebuilt.bots[0]?.archivedAt, null);
assert.deepEqual(rebuilt.groups, []);
diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts
index fce712331..cc76e5a13 100644
--- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts
+++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts
@@ -603,6 +603,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
sandbox: event.payload.sandbox,
runtimeMode: event.payload.runtimeMode,
usageCap: event.payload.usageCap,
+ voiceEnabled: event.payload.voiceEnabled,
groupId: event.payload.groupId,
archivedAt: null,
createdAt: event.payload.createdAt,
@@ -630,6 +631,9 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
? { runtimeMode: event.payload.runtimeMode }
: {}),
...(event.payload.usageCap !== undefined ? { usageCap: event.payload.usageCap } : {}),
+ ...(event.payload.voiceEnabled !== undefined
+ ? { voiceEnabled: event.payload.voiceEnabled }
+ : {}),
...(event.payload.groupId !== undefined ? { groupId: event.payload.groupId } : {}),
updatedAt: event.payload.updatedAt,
});
diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts
index 59e0e69ab..6d8a086a0 100644
--- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts
+++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts
@@ -92,6 +92,7 @@ const ProjectionBotDbRowSchema = ProjectionBot.mapFields(
engine: Schema.NullOr(Schema.fromJsonString(BotEngine)),
usageCap: Schema.NullOr(Schema.fromJsonString(BotUsageCap)),
disabledMcpServerIds: Schema.fromJsonString(Schema.Array(McpServerId)),
+ voiceEnabled: Schema.Number,
}),
);
const ProjectionGroupDbRowSchema = ProjectionGroup.mapFields(
@@ -371,6 +372,7 @@ function mapBotRow(row: Schema.Schema.Type<typeof ProjectionBotDbRowSchema>): Or
sandbox: row.sandbox,
runtimeMode: row.runtimeMode,
usageCap: row.usageCap,
+ voiceEnabled: row.voiceEnabled === 1,
groupId: row.groupId,
archivedAt: row.archivedAt,
createdAt: row.createdAt,
@@ -480,8 +482,8 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {
bot_id AS "botId", name, title, label, description,
disabled_mcp_server_ids_json AS "disabledMcpServerIds", avatar_json AS "avatar",
engine_json AS "engine", sandbox, runtime_mode AS "runtimeMode",
- usage_cap_json AS "usageCap", group_id AS "groupId", archived_at AS "archivedAt",
- created_at AS "createdAt", updated_at AS "updatedAt"
+ usage_cap_json AS "usageCap", voice_enabled AS "voiceEnabled", group_id AS "groupId",
+ archived_at AS "archivedAt", created_at AS "createdAt", updated_at AS "updatedAt"
FROM projection_bots
ORDER BY created_at ASC, bot_id ASC
`,
diff --git a/apps/server/src/orchestration/decider.groups.test.ts b/apps/server/src/orchestration/decider.groups.test.ts
index af163d41d..99a1b4f06 100644
--- a/apps/server/src/orchestration/decider.groups.test.ts
+++ b/apps/server/src/orchestration/decider.groups.test.ts
@@ -46,6 +46,7 @@ function makeBot(input: {
sandbox: "local",
runtimeMode: "full-access",
usageCap: null,
+ voiceEnabled: false,
groupId: input.groupId ?? null,
archivedAt: input.archivedAt ?? null,
createdAt: NOW,
diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts
index 95743471c..bae56647d 100644
--- a/apps/server/src/orchestration/decider.ts
+++ b/apps/server/src/orchestration/decider.ts
@@ -411,6 +411,7 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand"
sandbox: command.sandbox,
runtimeMode: command.runtimeMode,
usageCap: command.usageCap,
+ voiceEnabled: command.voiceEnabled ?? false,
groupId: command.groupId,
createdAt: command.createdAt,
updatedAt: command.createdAt,
@@ -518,6 +519,7 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand"
...(command.sandbox !== undefined ? { sandbox: command.sandbox } : {}),
...(command.runtimeMode !== undefined ? { runtimeMode: command.runtimeMode } : {}),
...(command.usageCap !== undefined ? { usageCap: command.usageCap } : {}),
+ ...(command.voiceEnabled !== undefined ? { voiceEnabled: command.voiceEnabled } : {}),
...(command.groupId !== undefined ? { groupId: command.groupId } : {}),
updatedAt: occurredAt,
},
@@ -1448,6 +1450,39 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand"
};
}
+ case "thread.voice-transcript.append": {
+ yield* requireThread({
+ readModel,
+ command,
+ threadId: command.threadId,
+ });
+ if (command.respondingBotId !== undefined) {
+ yield* requireBot({ readModel, command, botId: command.respondingBotId });
+ }
+ return {
+ ...(yield* withEventBase({
+ aggregateKind: "thread",
+ aggregateId: command.threadId,
+ occurredAt: command.createdAt,
+ commandId: command.commandId,
+ })),
+ type: "thread.message-sent",
+ payload: {
+ threadId: command.threadId,
+ messageId: command.messageId,
+ role: command.role,
+ text: command.text,
+ turnId: null,
+ ...(command.respondingBotId !== undefined
+ ? { respondingBotId: command.respondingBotId }
+ : {}),
+ streaming: false,
+ createdAt: command.createdAt,
+ updatedAt: command.createdAt,
+ },
+ };
+ }
+
case "thread.meta.update": {
const thread = yield* requireThread({
readModel,
diff --git a/apps/server/src/orchestration/decider.voiceTranscript.test.ts b/apps/server/src/orchestration/decider.voiceTranscript.test.ts
new file mode 100644
index 000000000..0af1a4982
--- /dev/null
+++ b/apps/server/src/orchestration/decider.voiceTranscript.test.ts
@@ -0,0 +1,123 @@
+import {
+ BotId,
+ CommandId,
+ MessageId,
+ ProjectId,
+ ProviderInstanceId,
+ ThreadId,
+ type OrchestrationBot,
+ type OrchestrationReadModel,
+ type OrchestrationThread,
+} from "@t3tools/contracts";
+import { expect, it } from "@effect/vitest";
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import * as Effect from "effect/Effect";
+
+import { decideOrchestrationCommand } from "./decider.ts";
+import { createEmptyReadModel } from "./projector.ts";
+
+const NOW = "2026-08-27T12:00:00.000Z";
+const BOT_ID = BotId.make("bot-akeru");
+const THREAD_ID = ThreadId.make("thread-voice");
+
+function makeBot(): OrchestrationBot {
+ return {
+ id: BOT_ID,
+ name: "Akeru",
+ title: "Agent",
+ label: null,
+ description: null,
+ disabledMcpServerIds: [],
+ avatar: { kind: "dither", seed: BOT_ID },
+ engine: null,
+ sandbox: "local",
+ runtimeMode: "full-access",
+ usageCap: null,
+ voiceEnabled: true,
+ groupId: null,
+ archivedAt: null,
+ createdAt: NOW,
+ updatedAt: NOW,
+ };
+}
+
+function makeThread(): OrchestrationThread {
+ return {
+ id: THREAD_ID,
+ projectId: ProjectId.make("project-1"),
+ botId: BOT_ID,
+ groupId: null,
+ respondingBotId: null,
+ title: "Voice thread",
+ modelSelection: { instanceId: ProviderInstanceId.make("default"), model: "default-model" },
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ branch: null,
+ worktreePath: null,
+ latestTurn: null,
+ createdAt: NOW,
+ updatedAt: NOW,
+ archivedAt: null,
+ settledOverride: null,
+ settledAt: null,
+ deletedAt: null,
+ messages: [],
+ proposedPlans: [],
+ activities: [],
+ checkpoints: [],
+ session: null,
+ };
+}
+
+function makeReadModel(): OrchestrationReadModel {
+ return {
+ ...createEmptyReadModel(NOW),
+ bots: [makeBot()],
+ threads: [makeThread()],
+ };
+}
+
+const appendCommand = (overrides?: { readonly threadId?: ThreadId }) => ({
+ type: "thread.voice-transcript.append" as const,
+ commandId: CommandId.make("cmd-voice-transcript"),
+ threadId: overrides?.threadId ?? THREAD_ID,
+ messageId: MessageId.make("message-voice-1"),
+ role: "assistant" as const,
+ text: "I will start on that now.",
+ respondingBotId: BOT_ID,
+ createdAt: NOW,
+});
+
+it.layer(NodeServices.layer)("voice transcript decider", (it) => {
+ it.effect("appends a transcript message without starting a turn", () =>
+ Effect.gen(function* () {
+ const decided = yield* decideOrchestrationCommand({
+ command: appendCommand(),
+ readModel: makeReadModel(),
+ });
+ const events = Array.isArray(decided) ? decided : [decided];
+ expect(events).toHaveLength(1);
+ expect(events[0]).toMatchObject({
+ type: "thread.message-sent",
+ payload: {
+ threadId: THREAD_ID,
+ role: "assistant",
+ text: "I will start on that now.",
+ turnId: null,
+ respondingBotId: BOT_ID,
+ streaming: false,
+ },
+ });
+ }),
+ );
+
+ it.effect("rejects a transcript for an unknown thread", () =>
+ Effect.gen(function* () {
+ const error = yield* decideOrchestrationCommand({
+ command: appendCommand({ threadId: ThreadId.make("thread-missing") }),
+ readModel: makeReadModel(),
+ }).pipe(Effect.flip);
+ expect(String(error)).toContain("thread-missing");
+ }),
+ );
+});
diff --git a/apps/server/src/orchestration/projector.ts b/apps/server/src/orchestration/projector.ts
index 7239811d6..f87dbeae8 100644
--- a/apps/server/src/orchestration/projector.ts
+++ b/apps/server/src/orchestration/projector.ts
@@ -332,6 +332,7 @@ export function projectEvent(
sandbox: payload.sandbox,
runtimeMode: payload.runtimeMode,
usageCap: payload.usageCap,
+ voiceEnabled: payload.voiceEnabled,
groupId: payload.groupId,
archivedAt: null,
createdAt: payload.createdAt,
@@ -364,6 +365,7 @@ export function projectEvent(
...(payload.sandbox !== undefined ? { sandbox: payload.sandbox } : {}),
...(payload.runtimeMode !== undefined ? { runtimeMode: payload.runtimeMode } : {}),
...(payload.usageCap !== undefined ? { usageCap: payload.usageCap } : {}),
+ ...(payload.voiceEnabled !== undefined ? { voiceEnabled: payload.voiceEnabled } : {}),
...(payload.groupId !== undefined ? { groupId: payload.groupId } : {}),
updatedAt: payload.updatedAt,
}),
diff --git a/apps/server/src/persistence/Layers/ProjectionBots.ts b/apps/server/src/persistence/Layers/ProjectionBots.ts
index 05e668a2c..eb4224a92 100644
--- a/apps/server/src/persistence/Layers/ProjectionBots.ts
+++ b/apps/server/src/persistence/Layers/ProjectionBots.ts
@@ -1,6 +1,7 @@
import { BotAvatar, BotEngine, BotUsageCap, McpServerId } from "@t3tools/contracts";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
import * as Schema from "effect/Schema";
import * as Struct from "effect/Struct";
import * as SqlClient from "effect/unstable/sql/SqlClient";
@@ -20,9 +21,14 @@ const ProjectionBotDbRow = ProjectionBot.mapFields(
engine: Schema.NullOr(Schema.fromJsonString(BotEngine)),
usageCap: Schema.NullOr(Schema.fromJsonString(BotUsageCap)),
disabledMcpServerIds: Schema.fromJsonString(Schema.Array(McpServerId)),
+ voiceEnabled: Schema.Number,
}),
);
+function toProjectionBot(row: Schema.Schema.Type<typeof ProjectionBotDbRow>): ProjectionBot {
+ return { ...row, voiceEnabled: row.voiceEnabled === 1 };
+}
+
const makeProjectionBotRepository = Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
@@ -31,14 +37,15 @@ const makeProjectionBotRepository = Effect.gen(function* () {
execute: (row) => sql`
INSERT INTO projection_bots (
bot_id, name, title, label, description, disabled_mcp_server_ids_json,
- avatar_json, engine_json, sandbox, runtime_mode, usage_cap_json,
+ avatar_json, engine_json, sandbox, runtime_mode, usage_cap_json, voice_enabled,
group_id, archived_at, created_at, updated_at
) VALUES (
${row.botId}, ${row.name}, ${row.title}, ${row.label}, ${row.description},
${JSON.stringify(row.disabledMcpServerIds)}, ${JSON.stringify(row.avatar)},
${row.engine === null ? null : JSON.stringify(row.engine)}, ${row.sandbox},
${row.runtimeMode}, ${row.usageCap === null ? null : JSON.stringify(row.usageCap)},
- ${row.groupId}, ${row.archivedAt}, ${row.createdAt}, ${row.updatedAt}
+ ${row.voiceEnabled ? 1 : 0}, ${row.groupId}, ${row.archivedAt}, ${row.createdAt},
+ ${row.updatedAt}
)
ON CONFLICT (bot_id) DO UPDATE SET
name = excluded.name,
@@ -51,6 +58,7 @@ const makeProjectionBotRepository = Effect.gen(function* () {
sandbox = excluded.sandbox,
runtime_mode = excluded.runtime_mode,
usage_cap_json = excluded.usage_cap_json,
+ voice_enabled = excluded.voice_enabled,
group_id = excluded.group_id,
archived_at = excluded.archived_at,
created_at = excluded.created_at,
@@ -66,8 +74,8 @@ const makeProjectionBotRepository = Effect.gen(function* () {
bot_id AS "botId", name, title, label, description,
disabled_mcp_server_ids_json AS "disabledMcpServerIds", avatar_json AS "avatar",
engine_json AS "engine", sandbox, runtime_mode AS "runtimeMode",
- usage_cap_json AS "usageCap", group_id AS "groupId", archived_at AS "archivedAt",
- created_at AS "createdAt", updated_at AS "updatedAt"
+ usage_cap_json AS "usageCap", voice_enabled AS "voiceEnabled", group_id AS "groupId",
+ archived_at AS "archivedAt", created_at AS "createdAt", updated_at AS "updatedAt"
FROM projection_bots
WHERE bot_id = ${botId}
`,
@@ -81,8 +89,8 @@ const makeProjectionBotRepository = Effect.gen(function* () {
bot_id AS "botId", name, title, label, description,
disabled_mcp_server_ids_json AS "disabledMcpServerIds", avatar_json AS "avatar",
engine_json AS "engine", sandbox, runtime_mode AS "runtimeMode",
- usage_cap_json AS "usageCap", group_id AS "groupId", archived_at AS "archivedAt",
- created_at AS "createdAt", updated_at AS "updatedAt"
+ usage_cap_json AS "usageCap", voice_enabled AS "voiceEnabled", group_id AS "groupId",
+ archived_at AS "archivedAt", created_at AS "createdAt", updated_at AS "updatedAt"
FROM projection_bots
ORDER BY created_at ASC, bot_id ASC
`,
@@ -94,10 +102,12 @@ const makeProjectionBotRepository = Effect.gen(function* () {
);
const getById: ProjectionBotRepositoryShape["getById"] = (input) =>
getBotRow(input).pipe(
+ Effect.map(Option.map(toProjectionBot)),
Effect.mapError(toPersistenceSqlError("ProjectionBotRepository.getById:query")),
);
const listAll: ProjectionBotRepositoryShape["listAll"] = () =>
listBotRows(undefined).pipe(
+ Effect.map((rows) => rows.map(toProjectionBot)),
Effect.mapError(toPersistenceSqlError("ProjectionBotRepository.listAll:query")),
);
diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts
index e7fd3a368..1a8455d9e 100644
--- a/apps/server/src/persistence/Migrations.ts
+++ b/apps/server/src/persistence/Migrations.ts
@@ -65,6 +65,7 @@ import Migration0049 from "./Migrations/049_BotRuntimeModeAndUsageCap.ts";
import Migration0050 from "./Migrations/050_BotProfileMetadata.ts";
import Migration0051 from "./Migrations/051_BotDisabledMcpServers.ts";
import Migration0052 from "./Migrations/052_ExecutorPluginCommand.ts";
+import Migration0053 from "./Migrations/053_BotVoiceEnabled.ts";
/**
* Migration loader with all migrations defined inline.
@@ -129,6 +130,7 @@ export const migrationEntries = [
[50, "BotProfileMetadata", Migration0050],
[51, "BotDisabledMcpServers", Migration0051],
[52, "ExecutorPluginCommand", Migration0052],
+ [53, "BotVoiceEnabled", Migration0053],
] as const;
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
diff --git a/apps/server/src/persistence/Migrations/046_049_AkeruProjectionMigrations.test.ts b/apps/server/src/persistence/Migrations/046_049_AkeruProjectionMigrations.test.ts
index a9d144361..15ea8af11 100644
--- a/apps/server/src/persistence/Migrations/046_049_AkeruProjectionMigrations.test.ts
+++ b/apps/server/src/persistence/Migrations/046_049_AkeruProjectionMigrations.test.ts
@@ -13,7 +13,7 @@ memoryLayer()("Akeru projection migration slots", (it) => {
Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
- assert.deepEqual(migrationManifest.slice(-8), [
+ assert.deepEqual(migrationManifest.slice(-9), [
[45, "ProjectionBotsAndGroups"],
[46, "ProjectionThreadOwnership"],
[47, "ProjectionMcpServers"],
@@ -22,6 +22,7 @@ memoryLayer()("Akeru projection migration slots", (it) => {
[50, "BotProfileMetadata"],
[51, "BotDisabledMcpServers"],
[52, "ExecutorPluginCommand"],
+ [53, "BotVoiceEnabled"],
]);
yield* runMigrations({ toMigrationInclusive: 45 });
diff --git a/apps/server/src/persistence/Migrations/053_BotVoiceEnabled.test.ts b/apps/server/src/persistence/Migrations/053_BotVoiceEnabled.test.ts
new file mode 100644
index 000000000..f5b047058
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/053_BotVoiceEnabled.test.ts
@@ -0,0 +1,26 @@
+import { assert, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as SqlClient from "effect/unstable/sql/SqlClient";
+
+import { runMigrations } from "../Migrations.ts";
+import * as NodeSqliteClient from "../NodeSqliteClient.ts";
+
+it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory()))("053_BotVoiceEnabled", (it) => {
+ it.effect("adds voice as an off-by-default bot setting", () =>
+ Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ yield* runMigrations({ toMigrationInclusive: 52 });
+ yield* runMigrations({ toMigrationInclusive: 53 });
+
+ const columns = yield* sql<{
+ readonly name: string;
+ readonly notnull: number;
+ readonly dflt_value: string | null;
+ }>`PRAGMA table_info(projection_bots)`;
+ const voice = columns.find((column) => column.name === "voice_enabled");
+ assert.equal(voice?.notnull, 1);
+ assert.equal(voice?.dflt_value, "0");
+ }),
+ );
+});
diff --git a/apps/server/src/persistence/Migrations/053_BotVoiceEnabled.ts b/apps/server/src/persistence/Migrations/053_BotVoiceEnabled.ts
new file mode 100644
index 000000000..8827957f5
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/053_BotVoiceEnabled.ts
@@ -0,0 +1,14 @@
+import * as Effect from "effect/Effect";
+import * as SqlClient from "effect/unstable/sql/SqlClient";
+
+export default Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ const columns = yield* sql<{ readonly name: string }>`PRAGMA table_info(projection_bots)`;
+
+ if (!columns.some((column) => column.name === "voice_enabled")) {
+ yield* sql`
+ ALTER TABLE projection_bots
+ ADD COLUMN voice_enabled INTEGER NOT NULL DEFAULT 0
+ `;
+ }
+});
diff --git a/apps/server/src/persistence/Services/ProjectionBots.ts b/apps/server/src/persistence/Services/ProjectionBots.ts
index 4c5ed920d..fb31d53b3 100644
--- a/apps/server/src/persistence/Services/ProjectionBots.ts
+++ b/apps/server/src/persistence/Services/ProjectionBots.ts
@@ -28,6 +28,7 @@ export const ProjectionBot = Schema.Struct({
sandbox: Schema.NullOr(BotSandbox),
runtimeMode: RuntimeMode,
usageCap: Schema.NullOr(BotUsageCap),
+ voiceEnabled: Schema.Boolean,
groupId: Schema.NullOr(GroupId),
archivedAt: Schema.NullOr(IsoDateTime),
createdAt: IsoDateTime,
diff --git a/apps/server/src/provider/Layers/AgentController.test.ts b/apps/server/src/provider/Layers/AgentController.test.ts
index 5547f3958..a36a19943 100644
--- a/apps/server/src/provider/Layers/AgentController.test.ts
+++ b/apps/server/src/provider/Layers/AgentController.test.ts
@@ -155,7 +155,7 @@ function makeMastraHarness() {
respondToToolApproval: vi.fn(),
respondToToolSuspension: vi.fn(async () => undefined),
} as unknown as Session<Record<string, unknown>>;
- const createSession = vi.fn(async () => session as never);
+ const createSession = vi.fn(async (_input: unknown) => session as never);
const deleteSession = vi.fn(async () => true);
const destroy = vi.fn(async () => undefined);
const factory: NonNullable<AgentControllerLiveOptions["makeMastraHarness"]> = async (options) => {
diff --git a/apps/server/src/provider/Layers/AgentController.ts b/apps/server/src/provider/Layers/AgentController.ts
index 96f3c6ac1..87b82cdcd 100644
--- a/apps/server/src/provider/Layers/AgentController.ts
+++ b/apps/server/src/provider/Layers/AgentController.ts
@@ -566,8 +566,8 @@ const make = (options?: AgentControllerLiveOptions) =>
const workspace = yield* runMastra("workspace.create", () =>
createBotWorkspace({
threadId: key,
- cwd: input.cwd,
- sandbox: input.botSandbox,
+ ...(input.cwd ? { cwd: input.cwd } : {}),
+ ...(input.botSandbox !== undefined ? { sandbox: input.botSandbox } : {}),
...(options?.makeRemoteWorkspace
? { makeRemoteWorkspace: options.makeRemoteWorkspace }
: {}),
diff --git a/apps/server/src/subscription-auth/service.ts b/apps/server/src/subscription-auth/service.ts
index 43b8dfaf7..630686712 100644
--- a/apps/server/src/subscription-auth/service.ts
+++ b/apps/server/src/subscription-auth/service.ts
@@ -378,6 +378,18 @@ export class SubscriptionAuthService {
return refresh;
}
+ async getOpenAICodexAccess(): Promise<
+ { readonly accessToken: string; readonly accountId: string } | undefined
+ > {
+ this.reload();
+ const accessToken = await this.getAccessToken("openai-codex");
+ const credential = this.data["openai-codex"];
+ const accountId = credential?.accountId;
+ return accessToken && typeof accountId === "string" && accountId.length > 0
+ ? { accessToken, accountId }
+ : undefined;
+ }
+
private async refreshCredential(
provider: SubscriptionProviderId,
credential: OAuthCredential,
diff --git a/apps/server/src/voiceCall/VoiceCallManager.test.ts b/apps/server/src/voiceCall/VoiceCallManager.test.ts
new file mode 100644
index 000000000..3bc475e2c
--- /dev/null
+++ b/apps/server/src/voiceCall/VoiceCallManager.test.ts
@@ -0,0 +1,288 @@
+import { BotId, VoiceCallError } from "@t3tools/contracts";
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { assert, it } from "@effect/vitest";
+import { vi } from "vite-plus/test";
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+
+import { ServerConfig } from "../config.ts";
+import { ServerSettingsService } from "../serverSettings.ts";
+import {
+ ProjectionBotRepository,
+ type ProjectionBotRepositoryShape,
+} from "../persistence/Services/ProjectionBots.ts";
+import { defaultSession, parseCodexCliAuth, VoiceCallManager, layer } from "./VoiceCallManager.ts";
+
+it("accepts a Codex CLI auth file without auth_mode", () => {
+ assert.deepEqual(
+ parseCodexCliAuth(JSON.stringify({ tokens: { access_token: "token", account_id: "account" } })),
+ { accessToken: "token", accountId: "account" },
+ );
+ assert.isUndefined(parseCodexCliAuth("not json"));
+});
+
+it("creates a pinned realtime session through the ChatGPT subscription endpoint", async () => {
+ const fetchMock = vi.fn(async (url: string | URL | Request, init?: RequestInit) => {
+ assert.equal(String(url), "https://chatgpt.com/backend-api/codex/realtime/calls");
+ assert.isNull(new Headers(init?.headers).get("OpenAI-Alpha"));
+ const body = JSON.parse(String(init?.body)) as {
+ readonly session: {
+ readonly type: string;
+ readonly model: string;
+ readonly audio: {
+ readonly input: { readonly turn_detection: { readonly type: string } };
+ readonly output: { readonly voice: string };
+ };
+ };
+ };
+ assert.equal(body.session.type, "realtime");
+ assert.equal(body.session.model, "gpt-realtime-2");
+ assert.equal(body.session.audio.output.voice, "marin");
+ assert.equal(body.session.audio.input.turn_detection.type, "semantic_vad");
+ return new Response("answer-sdp");
+ });
+ vi.stubGlobal("fetch", fetchMock);
+ try {
+ const answer = await defaultSession().negotiate({
+ offerSdp: "offer-sdp",
+ instructions: "Be concise.",
+ accessToken: "access",
+ accountId: "account",
+ voice: "marin",
+ signal: new AbortController().signal,
+ });
+ assert.equal(answer, "answer-sdp");
+ } finally {
+ vi.unstubAllGlobals();
+ }
+});
+
+const botId = BotId.make("bot-voice");
+const now = "2026-08-27T00:00:00.000Z";
+const bot = {
+ botId,
+ name: "Akeru",
+ title: "Generalist",
+ label: null,
+ description: "Helps Leo get work done.",
+ disabledMcpServerIds: [],
+ avatar: { kind: "blob" as const, shape: "circle" as const, color: "#5B7FD4" },
+ engine: null,
+ sandbox: null,
+ runtimeMode: "full-access" as const,
+ usageCap: null,
+ voiceEnabled: true,
+ groupId: null,
+ archivedAt: null,
+ createdAt: now,
+ updatedAt: now,
+};
+
+const repository = {
+ upsert: () => Effect.void,
+ getById: ({ botId: requested }) =>
+ Effect.succeed(requested === botId ? Option.some(bot) : Option.none()),
+ listAll: () => Effect.succeed([bot]),
+} satisfies ProjectionBotRepositoryShape;
+
+const TestLayer = layer({
+ getCredential: async () => ({ accessToken: "access", accountId: "account" }),
+ makeSession: () => ({ negotiate: async () => "answer-sdp" }),
+}).pipe(
+ Layer.provide(Layer.succeed(ProjectionBotRepository, repository)),
+ Layer.provide(ServerSettingsService.layerTest()),
+ Layer.provide(
+ ServerConfig.layerTest(process.cwd(), { prefix: "akeru-voice-call-test-" }).pipe(
+ Layer.provide(NodeServices.layer),
+ ),
+ ),
+);
+
+const VoiceDisabledTestLayer = layer({
+ getCredential: async () => ({ accessToken: "access", accountId: "account" }),
+ makeSession: () => ({ negotiate: async () => "answer-sdp" }),
+}).pipe(
+ Layer.provide(Layer.succeed(ProjectionBotRepository, repository)),
+ Layer.provide(ServerSettingsService.layerTest({ voice: { enabled: false } })),
+ Layer.provide(
+ ServerConfig.layerTest(process.cwd(), { prefix: "akeru-voice-call-disabled-test-" }).pipe(
+ Layer.provide(NodeServices.layer),
+ ),
+ ),
+);
+
+let negotiatedVoice: string | undefined;
+const SelectedVoiceTestLayer = layer({
+ getCredential: async () => ({ accessToken: "access", accountId: "account" }),
+ makeSession: () => ({
+ negotiate: async ({ voice }) => {
+ negotiatedVoice = voice;
+ return "answer-sdp";
+ },
+ }),
+}).pipe(
+ Layer.provide(Layer.succeed(ProjectionBotRepository, repository)),
+ Layer.provide(ServerSettingsService.layerTest({ voice: { voice: "cedar" } })),
+ Layer.provide(
+ ServerConfig.layerTest(process.cwd(), { prefix: "akeru-voice-call-selection-test-" }).pipe(
+ Layer.provide(NodeServices.layer),
+ ),
+ ),
+);
+
+it.layer(SelectedVoiceTestLayer)("VoiceCallManager voice selection", (it) => {
+ it.effect("uses the globally selected voice for new calls", () =>
+ Effect.gen(function* () {
+ negotiatedVoice = undefined;
+ const manager = yield* VoiceCallManager;
+ const result = yield* manager.start({ botId, sdp: "offer-sdp" }, "client-1");
+ assert.equal(negotiatedVoice, "cedar");
+ yield* manager.hangup(result.call.callId, "client-1");
+ }),
+ );
+});
+
+it.layer(VoiceDisabledTestLayer)("VoiceCallManager global settings", (it) => {
+ it.effect("blocks every bot when voice is disabled globally", () =>
+ Effect.gen(function* () {
+ const manager = yield* VoiceCallManager;
+ const result = yield* Effect.result(manager.start({ botId, sdp: "offer-sdp" }, "client-1"));
+ assert.equal(result._tag, "Failure");
+ if (result._tag === "Failure") {
+ assert.equal(result.failure.reason, "voice-disabled");
+ assert.equal(result.failure.message, "Voice calls are disabled in Settings.");
+ }
+ }),
+ );
+});
+
+it.layer(TestLayer)("VoiceCallManager", (it) => {
+ it.effect("refuses hangup from a different connection owner", () =>
+ Effect.gen(function* () {
+ const manager = yield* VoiceCallManager;
+ const first = yield* manager.start({ botId, sdp: "offer-sdp" }, "client-1");
+
+ const stolen = yield* Effect.result(manager.hangup(first.call.callId, "client-2"));
+ assert.equal(stolen._tag, "Failure");
+ if (stolen._tag === "Failure") {
+ assert.equal(stolen.failure.reason, "call-not-active");
+ }
+ assert.equal((yield* manager.get).status, "live");
+
+ yield* manager.hangupOwner("client-2");
+ assert.equal((yield* manager.get).status, "live");
+
+ yield* manager.hangup(first.call.callId, "client-1");
+ assert.deepEqual(yield* manager.get, { status: "idle" });
+ }),
+ );
+
+ it.effect("refuses a second call until hangup and clears the active call", () =>
+ Effect.gen(function* () {
+ const manager = yield* VoiceCallManager;
+ const first = yield* manager.start({ botId, sdp: "offer-sdp" }, "client-1");
+ assert.equal(first.call.status, "live");
+ assert.equal(first.answerSdp, "answer-sdp");
+
+ const second = yield* Effect.result(
+ manager.start({ botId, sdp: "second-offer" }, "client-2"),
+ );
+ assert.equal(second._tag, "Failure");
+ if (second._tag === "Failure") {
+ const failure = second.failure;
+ assert.instanceOf(failure, VoiceCallError);
+ assert.equal(failure.reason, "already-active");
+ }
+
+ assert.deepEqual(yield* manager.hangup(first.call.callId, "client-1"), {
+ status: "idle",
+ });
+ assert.deepEqual(yield* manager.get, { status: "idle" });
+
+ const restarted = yield* manager.start({ botId, sdp: "third-offer" }, "client-1");
+ assert.equal(restarted.call.status, "live");
+ yield* manager.hangupOwner("client-1");
+ assert.deepEqual(yield* manager.get, { status: "idle" });
+ }),
+ );
+});
+
+let codexCliCredentialRequested = false;
+const CodexCliCredentialTestLayer = layer({
+ getCodexCliCredential: async () => {
+ codexCliCredentialRequested = true;
+ return { accessToken: "codex-access", accountId: "codex-account" };
+ },
+ makeSession: () => ({
+ negotiate: async ({ accessToken, accountId }) =>
+ accessToken === "codex-access" && accountId === "codex-account"
+ ? "answer-sdp"
+ : Promise.reject(new Error("wrong credential")),
+ }),
+}).pipe(
+ Layer.provide(Layer.succeed(ProjectionBotRepository, repository)),
+ Layer.provide(ServerSettingsService.layerTest()),
+ Layer.provide(
+ ServerConfig.layerTest(process.cwd(), { prefix: "akeru-voice-call-codex-auth-test-" }).pipe(
+ Layer.provide(NodeServices.layer),
+ ),
+ ),
+);
+
+it.layer(CodexCliCredentialTestLayer)("VoiceCallManager Codex CLI auth", (it) => {
+ it.effect("uses the existing ChatGPT subscription when app auth is not connected", () =>
+ Effect.gen(function* () {
+ codexCliCredentialRequested = false;
+ const manager = yield* VoiceCallManager;
+ const result = yield* manager.start({ botId, sdp: "offer-sdp" }, "client-1");
+ assert.isTrue(codexCliCredentialRequested);
+ assert.equal(result.answerSdp, "answer-sdp");
+ }),
+ );
+});
+
+let negotiationStarted: (() => void) | undefined;
+const PendingTestLayer = layer({
+ getCredential: async () => ({ accessToken: "access", accountId: "account" }),
+ makeSession: () => ({
+ negotiate: ({ signal }) =>
+ new Promise<string>((_resolve, reject) => {
+ negotiationStarted?.();
+ signal.addEventListener("abort", () => reject(new Error("aborted")), { once: true });
+ }),
+ }),
+}).pipe(
+ Layer.provide(Layer.succeed(ProjectionBotRepository, repository)),
+ Layer.provide(ServerSettingsService.layerTest()),
+ Layer.provide(
+ ServerConfig.layerTest(process.cwd(), { prefix: "akeru-voice-call-pending-test-" }).pipe(
+ Layer.provide(NodeServices.layer),
+ ),
+ ),
+);
+
+it.layer(PendingTestLayer)("VoiceCallManager pending start", (it) => {
+ it.effect("cancels an in-flight negotiation when the call hangs up", () =>
+ Effect.gen(function* () {
+ const manager = yield* VoiceCallManager;
+ const started = new Promise<void>((resolve) => {
+ negotiationStarted = resolve;
+ });
+ const startFiber = yield* manager
+ .start({ botId, sdp: "offer-sdp" }, "client-1")
+ .pipe(Effect.result, Effect.forkChild);
+ yield* Effect.promise(() => started);
+ const pending = yield* manager.get;
+ assert.notEqual(pending.status, "idle");
+ if (pending.status === "idle") return;
+
+ yield* manager.hangup(pending.callId, "client-1");
+ const result = yield* Fiber.join(startFiber);
+ assert.equal(result._tag, "Failure");
+ if (result._tag === "Failure") assert.equal(result.failure.reason, "call-not-active");
+ assert.deepEqual(yield* manager.get, { status: "idle" });
+ }),
+ );
+});
diff --git a/apps/server/src/voiceCall/VoiceCallManager.ts b/apps/server/src/voiceCall/VoiceCallManager.ts
new file mode 100644
index 000000000..8ba713fca
--- /dev/null
+++ b/apps/server/src/voiceCall/VoiceCallManager.ts
@@ -0,0 +1,407 @@
+// @effect-diagnostics globalFetch:off nodeBuiltinImport:off
+import * as NodeCrypto from "node:crypto";
+import * as NodeFS from "node:fs/promises";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+
+import {
+ CHATGPT_REALTIME_VOICE_MODEL,
+ TrimmedNonEmptyString,
+ type ChatGptRealtimeVoice,
+ VoiceCallError,
+ type BotId,
+ type VoiceCallSnapshot,
+ type VoiceCallStartInput,
+ type VoiceCallStartResult,
+} from "@t3tools/contracts";
+import * as Context from "effect/Context";
+import * as DateTime from "effect/DateTime";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Semaphore from "effect/Semaphore";
+import * as Schema from "effect/Schema";
+
+import * as ServerConfig from "../config.ts";
+import { ProjectionBotRepository } from "../persistence/Services/ProjectionBots.ts";
+import { ServerSettingsService } from "../serverSettings.ts";
+import { SubscriptionAuthService } from "../subscription-auth/service.ts";
+
+const CHATGPT_REALTIME_CALL_URL = "https://chatgpt.com/backend-api/codex/realtime/calls";
+
+interface ActiveVoiceCall {
+ readonly callId: string;
+ readonly ownerId: string;
+ readonly botId: BotId;
+ readonly botName: string;
+ readonly startedAt: string;
+ readonly abortController: AbortController;
+ status: "starting" | "live";
+}
+
+const CodexCliAuth = Schema.Struct({
+ tokens: Schema.Struct({
+ access_token: TrimmedNonEmptyString,
+ account_id: TrimmedNonEmptyString,
+ }),
+});
+
+interface ChatGptRealtimeSession {
+ readonly negotiate: (input: {
+ readonly offerSdp: string;
+ readonly instructions: string;
+ readonly accessToken: string;
+ readonly accountId: string;
+ readonly voice: ChatGptRealtimeVoice;
+ readonly signal: AbortSignal;
+ }) => Promise<string>;
+}
+
+export interface VoiceCallManagerOptions {
+ readonly makeSession?: () => ChatGptRealtimeSession;
+ readonly getCredential?: () => Promise<
+ { readonly accessToken: string; readonly accountId: string } | undefined
+ >;
+ readonly getCodexCliCredential?: () => Promise<
+ { readonly accessToken: string; readonly accountId: string } | undefined
+ >;
+}
+
+export function parseCodexCliAuth(
+ encoded: string,
+): { readonly accessToken: string; readonly accountId: string } | undefined {
+ try {
+ const decoded = Schema.decodeUnknownSync(CodexCliAuth)(JSON.parse(encoded));
+ return { accessToken: decoded.tokens.access_token, accountId: decoded.tokens.account_id };
+ } catch {
+ return undefined;
+ }
+}
+
+async function getCodexCliCredential(): Promise<
+ { readonly accessToken: string; readonly accountId: string } | undefined
+> {
+ try {
+ const encoded = await NodeFS.readFile(
+ NodePath.join(NodeOS.homedir(), ".codex", "auth.json"),
+ "utf8",
+ );
+ return parseCodexCliAuth(encoded);
+ } catch {
+ return undefined;
+ }
+}
+
+export function defaultSession(): ChatGptRealtimeSession {
+ return {
+ negotiate: async ({ offerSdp, instructions, accessToken, accountId, voice, signal }) => {
+ const response = await fetch(CHATGPT_REALTIME_CALL_URL, {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ "ChatGPT-Account-ID": accountId,
+ "Content-Type": "application/json",
+ originator: "akeru",
+ "User-Agent": "akeru",
+ },
+ body: JSON.stringify({
+ sdp: offerSdp,
+ session: {
+ type: "realtime",
+ model: CHATGPT_REALTIME_VOICE_MODEL,
+ instructions,
+ audio: {
+ input: {
+ format: { type: "audio/pcm", rate: 24_000 },
+ transcription: { model: "gpt-4o-mini-transcribe" },
+ turn_detection: { type: "semantic_vad" },
+ },
+ output: { voice },
+ },
+ tools: [
+ {
+ type: "function",
+ name: "send_to_chat",
+ description:
+ "Delegate a task to the coding-agent team in the chat thread. The team can read files, run commands, write code, and use tools on the user's machine. Call this for any request that needs execution, then confirm the handoff by voice. Pass the user's request as one clear message.",
+ parameters: {
+ type: "object",
+ properties: {
+ message: {
+ type: "string",
+ description: "The user's request, cleaned up into one clear chat message.",
+ },
+ },
+ required: ["message"],
+ },
+ },
+ ],
+ tool_choice: "auto",
+ },
+ }),
+ signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]),
+ });
+ const answerSdp = await response.text();
+ if (!response.ok) {
+ const detail = answerSdp.trim().slice(0, 500);
+ throw new Error(
+ detail.length > 0
+ ? `ChatGPT realtime call failed with status ${response.status}: ${detail}`
+ : `ChatGPT realtime call failed with status ${response.status}.`,
+ );
+ }
+ if (answerSdp.trim().length === 0) {
+ throw new Error("ChatGPT realtime call returned an empty SDP answer.");
+ }
+ return answerSdp;
+ },
+ };
+}
+
+function snapshot(active: ActiveVoiceCall | null): VoiceCallSnapshot {
+ return active === null
+ ? { status: "idle" }
+ : {
+ callId: active.callId,
+ status: active.status,
+ botId: active.botId,
+ botName: active.botName,
+ startedAt: active.startedAt,
+ };
+}
+
+function instructionsForBot(bot: {
+ readonly name: string;
+ readonly title: string;
+ readonly description: string | null;
+}): string {
+ return [
+ `You are ${bot.name}, the user's ${bot.title}.`,
+ bot.description,
+ "Talk like a sharp teammate. Answer first. Use short, plain sentences.",
+ "You are a bot on the user's team. Do not claim to be a person.",
+ "You lead a team of coding agents that work in the chat thread. They can read files, run commands, write code, and use tools on the user's machine. You delegate; they execute.",
+ "For any task that needs files, tools, code, or the user's computer, call send_to_chat with the request as one clear message. Then tell the user you have handed it to the team. Never say the task is impossible for you.",
+ "Only answer directly for conversation, questions you can answer from context, and retelling what the team has already done.",
+ "Context items labeled 'Team update' describe work the chat team finished during this call. Use them when the user asks about progress.",
+ ]
+ .filter((part): part is string => typeof part === "string" && part.trim().length > 0)
+ .join("\n\n");
+}
+
+export class VoiceCallManager extends Context.Service<
+ VoiceCallManager,
+ {
+ readonly get: Effect.Effect<VoiceCallSnapshot>;
+ readonly start: (
+ input: VoiceCallStartInput,
+ ownerId: string,
+ ) => Effect.Effect<VoiceCallStartResult, VoiceCallError>;
+ readonly hangup: (
+ callId: string,
+ ownerId: string,
+ ) => Effect.Effect<VoiceCallSnapshot, VoiceCallError>;
+ readonly hangupOwner: (ownerId: string) => Effect.Effect<void>;
+ }
+>()("t3/voiceCall/VoiceCallManager") {}
+
+const make = (options?: VoiceCallManagerOptions) =>
+ Effect.gen(function* () {
+ const config = yield* ServerConfig.ServerConfig;
+ const bots = yield* ProjectionBotRepository;
+ const serverSettings = yield* ServerSettingsService;
+ const lock = yield* Semaphore.make(1);
+ const auth = SubscriptionAuthService.forSecretsDir(config.secretsDir);
+ let active: ActiveVoiceCall | null = null;
+
+ const clear = (callId: string) =>
+ lock.withPermits(1)(
+ Effect.sync(() => {
+ if (active?.callId !== callId) return;
+ active.abortController.abort();
+ active = null;
+ }),
+ );
+
+ const start = Effect.fn("VoiceCallManager.start")(function* (
+ input: VoiceCallStartInput,
+ ownerId: string,
+ ) {
+ const voiceSettings = yield* serverSettings.getSettings.pipe(
+ Effect.map((settings) => settings.voice),
+ Effect.mapError(
+ () =>
+ new VoiceCallError({
+ reason: "voice-disabled",
+ message: "Voice settings are unavailable.",
+ }),
+ ),
+ );
+ if (!voiceSettings.enabled) {
+ return yield* new VoiceCallError({
+ reason: "voice-disabled",
+ message: "Voice calls are disabled in Settings.",
+ });
+ }
+ const claimed = yield* lock.withPermits(1)(
+ Effect.gen(function* () {
+ if (active !== null) {
+ return yield* new VoiceCallError({
+ reason: "already-active",
+ message: `A call with ${active.botName} is already active. Hang up before starting another call.`,
+ });
+ }
+ const bot = yield* bots.getById({ botId: input.botId }).pipe(
+ Effect.mapError(
+ () =>
+ new VoiceCallError({
+ reason: "bot-not-found",
+ message: "Could not load this bot.",
+ }),
+ ),
+ );
+ if (Option.isNone(bot) || bot.value.archivedAt !== null) {
+ return yield* new VoiceCallError({
+ reason: "bot-not-found",
+ message: "This bot is not available.",
+ });
+ }
+ if (!bot.value.voiceEnabled) {
+ return yield* new VoiceCallError({
+ reason: "voice-disabled",
+ message: `Voice calls are disabled for ${bot.value.name}.`,
+ });
+ }
+ const startedAt = DateTime.formatIso(yield* DateTime.now);
+ const call: ActiveVoiceCall = {
+ callId: NodeCrypto.randomUUID(),
+ ownerId,
+ botId: bot.value.botId,
+ botName: bot.value.name,
+ startedAt,
+ abortController: new AbortController(),
+ status: "starting",
+ };
+ active = call;
+ return { call, bot: bot.value };
+ }),
+ );
+
+ const credential = yield* Effect.tryPromise({
+ try: () =>
+ options?.getCredential
+ ? options.getCredential()
+ : auth
+ .getOpenAICodexAccess()
+ .then(
+ (credential) =>
+ credential ?? (options?.getCodexCliCredential ?? getCodexCliCredential)(),
+ ),
+ catch: () =>
+ new VoiceCallError({
+ reason: "subscription-unavailable",
+ message: "Connect a ChatGPT subscription before starting a call.",
+ }),
+ }).pipe(Effect.tapError(() => clear(claimed.call.callId)));
+ if (credential === undefined) {
+ yield* clear(claimed.call.callId);
+ return yield* new VoiceCallError({
+ reason: "subscription-unavailable",
+ message: "Connect a ChatGPT subscription before starting a call.",
+ });
+ }
+
+ const session = (() => {
+ switch (voiceSettings.provider) {
+ case "chatgpt":
+ return (options?.makeSession ?? defaultSession)();
+ }
+ })();
+ const answerSdp = yield* Effect.tryPromise({
+ try: () =>
+ session.negotiate({
+ offerSdp: input.sdp,
+ instructions: instructionsForBot(claimed.bot),
+ signal: claimed.call.abortController.signal,
+ voice: voiceSettings.voice,
+ ...credential,
+ }),
+ catch: (cause) =>
+ active?.callId === claimed.call.callId
+ ? new VoiceCallError({
+ reason: "upstream-failed",
+ message:
+ cause instanceof Error
+ ? cause.message
+ : "Could not start the ChatGPT realtime call.",
+ })
+ : new VoiceCallError({
+ reason: "call-not-active",
+ message: "This call is no longer active.",
+ }),
+ }).pipe(Effect.tapError(() => clear(claimed.call.callId)));
+
+ yield* lock.withPermits(1)(
+ Effect.gen(function* () {
+ if (active?.callId !== claimed.call.callId) {
+ return yield* new VoiceCallError({
+ reason: "call-not-active",
+ message: "This call is no longer active.",
+ });
+ }
+ active.status = "live";
+ }),
+ );
+ return {
+ call: {
+ callId: claimed.call.callId,
+ status: "live" as const,
+ botId: claimed.call.botId,
+ botName: claimed.call.botName,
+ startedAt: claimed.call.startedAt,
+ },
+ answerSdp,
+ };
+ });
+
+ const hangup = (callId: string, ownerId: string) =>
+ lock.withPermits(1)(
+ Effect.gen(function* () {
+ if (active?.callId !== callId || active.ownerId !== ownerId) {
+ return yield* new VoiceCallError({
+ reason: "call-not-active",
+ message: "This call is no longer active.",
+ });
+ }
+ active.abortController.abort();
+ active = null;
+ return snapshot(active);
+ }),
+ );
+
+ const hangupOwner = (ownerId: string) =>
+ lock.withPermits(1)(
+ Effect.sync(() => {
+ if (active?.ownerId !== ownerId) return;
+ active.abortController.abort();
+ active = null;
+ }),
+ );
+
+ yield* Effect.addFinalizer(() =>
+ Effect.sync(() => {
+ active?.abortController.abort();
+ active = null;
+ }),
+ );
+
+ return VoiceCallManager.of({
+ get: Effect.sync(() => snapshot(active)),
+ start,
+ hangup,
+ hangupOwner,
+ });
+ });
+
+export const layer = (options?: VoiceCallManagerOptions) =>
+ Layer.effect(VoiceCallManager, make(options));
diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts
index 41a22f0fb..853fa9f3c 100644
--- a/apps/server/src/ws.ts
+++ b/apps/server/src/ws.ts
@@ -124,6 +124,7 @@ import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts
import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts";
import * as AnalyticsService from "./telemetry/AnalyticsService.ts";
import * as UsageService from "./usage/UsageService.ts";
+import * as VoiceCallManager from "./voiceCall/VoiceCallManager.ts";
import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";
import * as SourceControlDiscovery from "./sourceControl/SourceControlDiscovery.ts";
import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts";
@@ -429,11 +430,13 @@ const makeWsRpcLayer = (
currentSession: EnvironmentAuth.AuthenticatedSession,
clientOrigin: OrchestrationClientOrigin,
previewAutomationBroker: PreviewAutomationBroker.PreviewAutomationBroker["Service"],
+ voiceCalls: VoiceCallManager.VoiceCallManager["Service"],
) =>
WsRpcGroup.toLayer(
Effect.gen(function* () {
const currentSessionId = currentSession.sessionId;
const crypto = yield* Crypto.Crypto;
+ const voiceCallOwnerId = yield* crypto.randomUUIDv4.pipe(Effect.orDie);
const projectionSnapshotQuery = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;
const projectionBots = yield* ProjectionBots.ProjectionBotRepository;
const projectionGroups = yield* ProjectionGroups.ProjectionGroupRepository;
@@ -494,17 +497,21 @@ const makeWsRpcLayer = (
const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy;
const rpcClientIds = yield* Ref.make(new Set<RpcClientId>());
yield* Effect.addFinalizer(() =>
- Ref.get(rpcClientIds).pipe(
- Effect.flatMap((clientIds) =>
- Effect.forEach(
- clientIds,
- (clientId) => backgroundPolicy.removeRpcClient(currentSessionId, clientId),
- {
- discard: true,
- },
+ Effect.all(
+ [
+ Ref.get(rpcClientIds).pipe(
+ Effect.flatMap((clientIds) =>
+ Effect.forEach(
+ clientIds,
+ (clientId) => backgroundPolicy.removeRpcClient(currentSessionId, clientId),
+ { discard: true },
+ ),
+ ),
+ Effect.ignore,
),
- ),
- Effect.ignore,
+ voiceCalls.hangupOwner(voiceCallOwnerId),
+ ],
+ { discard: true },
),
);
const serverAuth = yield* EnvironmentAuth.EnvironmentAuth;
@@ -800,6 +807,7 @@ const makeWsRpcLayer = (
sandbox: nextBot.sandbox,
runtimeMode: nextBot.runtimeMode,
usageCap: nextBot.usageCap,
+ voiceEnabled: nextBot.voiceEnabled,
groupId: nextBot.groupId,
archivedAt: nextBot.archivedAt,
createdAt: nextBot.createdAt,
@@ -1854,6 +1862,20 @@ const makeWsRpcLayer = (
}),
{ "rpc.aggregate": "server" },
),
+ [WS_METHODS.voiceCallGet]: (_input) =>
+ observeRpcEffect(WS_METHODS.voiceCallGet, voiceCalls.get, {
+ "rpc.aggregate": "voice-call",
+ }),
+ [WS_METHODS.voiceCallStart]: (input) =>
+ observeRpcEffect(WS_METHODS.voiceCallStart, voiceCalls.start(input, voiceCallOwnerId), {
+ "rpc.aggregate": "voice-call",
+ }),
+ [WS_METHODS.voiceCallHangup]: ({ callId }) =>
+ observeRpcEffect(
+ WS_METHODS.voiceCallHangup,
+ voiceCalls.hangup(callId, voiceCallOwnerId),
+ { "rpc.aggregate": "voice-call" },
+ ),
[WS_METHODS.serverDiscoverSourceControl]: (_input) =>
observeRpcEffect(
WS_METHODS.serverDiscoverSourceControl,
@@ -2548,6 +2570,7 @@ const makeWsRpcLayer = (
export const websocketRpcRouteLayer = Layer.unwrap(
Effect.gen(function* () {
const previewAutomationBroker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
+ const voiceCalls = yield* VoiceCallManager.VoiceCallManager;
const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate;
return HttpRouter.add(
"GET",
@@ -2575,7 +2598,7 @@ export const websocketRpcRouteLayer = Layer.unwrap(
disableTracing: true,
}).pipe(
Effect.provide(
- makeWsRpcLayer(session, clientOrigin, previewAutomationBroker).pipe(
+ makeWsRpcLayer(session, clientOrigin, previewAutomationBroker, voiceCalls).pipe(
Layer.provideMerge(RpcSerialization.layerJson),
Layer.provide(ProviderMaintenanceRunner.layer),
Layer.provide(Layer.succeed(ServerSelfUpdate.ServerSelfUpdate, serverSelfUpdate)),
@@ -2616,4 +2639,4 @@ export const websocketRpcRouteLayer = Layer.unwrap(
),
);
}),
-);
+).pipe(Layer.provide(VoiceCallManager.layer()));
diff --git a/apps/web/src/components/chat/ChatHeader.tsx b/apps/web/src/components/chat/ChatHeader.tsx
index 03e677514..961609f05 100644
--- a/apps/web/src/components/chat/ChatHeader.tsx
+++ b/apps/web/src/components/chat/ChatHeader.tsx
@@ -34,6 +34,7 @@ import ProjectScriptsControl, {
import { OpenInPicker } from "./OpenInPicker";
import { type PresentationMode } from "./quietPresentation";
import { ActiveBotHeaderChip } from "../roster/ActiveBotHeaderChip";
+import { SelectedBotVoiceCallButton } from "../voice/VoiceCall";
import { useRemoteOpenState, type RemoteOpenMode } from "../../remoteOpen";
import { usePrimaryEnvironmentId } from "../../state/environments";
import { useT3ProjectFileScripts } from "~/hooks/useT3ProjectFileScripts";
@@ -387,6 +388,7 @@ export const ChatHeader = memo(function ChatHeader({
rightPanelOpen ? "pr-0" : "pr-16",
)}
>
+ <SelectedBotVoiceCallButton />
<Tooltip>
<TooltipTrigger
render={
diff --git a/apps/web/src/components/roster/BotDetailsPanel.test.tsx b/apps/web/src/components/roster/BotDetailsPanel.test.tsx
index 82922fdd6..17e924b07 100644
--- a/apps/web/src/components/roster/BotDetailsPanel.test.tsx
+++ b/apps/web/src/components/roster/BotDetailsPanel.test.tsx
@@ -19,6 +19,7 @@ const bot: Bot = {
sandbox: null,
runtimeMode: "full-access",
usageCap: null,
+ voiceEnabled: false,
groupId: null,
pinned: false,
archivedAt: null,
@@ -37,6 +38,8 @@ describe("BotDetailsPanel", () => {
expect(markup).toContain("Connect a provider");
expect(markup).toContain(">Sandbox</div>");
expect(markup).toContain('aria-label="Sandbox provider"');
+ expect(markup).toContain(">Voice calls</span>");
+ expect(markup).toContain('aria-label="Enable voice calls for Akeru"');
expect(markup).toContain(">Tools</div>");
expect(markup).toContain("No workspace tools");
expect(markup).toContain(">Manage</span>");
@@ -98,7 +101,7 @@ describe("BotDetailsPanel", () => {
);
expect(source).toContain("<BotDetailsPanel");
expect(source).toContain("onSaveBot=");
- expect(source).toContain("label, description, engine, sandbox, disabledMcpServerIds");
+ expect(source).toContain("voiceEnabled,");
expect(source).toContain("sandbox,");
expect(source).toContain("disabledMcpServerIds,");
expect(source).not.toContain("RightPanelTabs");
diff --git a/apps/web/src/components/roster/BotDetailsPanel.tsx b/apps/web/src/components/roster/BotDetailsPanel.tsx
index a2d5385a4..a41762386 100644
--- a/apps/web/src/components/roster/BotDetailsPanel.tsx
+++ b/apps/web/src/components/roster/BotDetailsPanel.tsx
@@ -37,6 +37,7 @@ import { Input } from "../ui/input";
import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
import { Sheet, SheetClose, SheetPopup, SheetTitle } from "../ui/sheet";
import { Textarea } from "../ui/textarea";
+import { Switch } from "../ui/switch";
import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
import { AvatarPickerDialog } from "./AvatarPickerDialog";
import { BotAvatarView } from "./BotAvatarView";
@@ -81,6 +82,7 @@ export interface BotProfileUpdate {
readonly description: string | null;
readonly engine: BotEngine | null;
readonly sandbox: Bot["sandbox"];
+ readonly voiceEnabled: boolean;
readonly disabledMcpServerIds: readonly McpServerId[];
}
@@ -104,6 +106,7 @@ function BotProfileEditor({
const [engineChanged, setEngineChanged] = useState(false);
const [toolsOpen, setToolsOpen] = useState(false);
const [sandbox, setSandbox] = useState<BotSandboxChoice>(() => botSandboxChoice(bot.sandbox));
+ const [voiceEnabled, setVoiceEnabled] = useState(bot.voiceEnabled);
const [disabledMcpServerIds, setDisabledMcpServerIds] = useState<readonly McpServerId[]>(
bot.disabledMcpServerIds,
);
@@ -162,6 +165,7 @@ function BotProfileEditor({
normalizedDescription !== bot.description ||
engineChanged ||
sandboxDirty ||
+ voiceEnabled !== bot.voiceEnabled ||
toolOverridesDirty;
const markChanged = () => setSaved(false);
@@ -271,6 +275,18 @@ function BotProfileEditor({
</Select>
</div>
+ <div className="flex min-h-10 items-center justify-between rounded-lg border border-border bg-muted/20 px-3">
+ <span className="text-sm font-medium">Voice calls</span>
+ <Switch
+ checked={voiceEnabled}
+ onCheckedChange={(checked) => {
+ setVoiceEnabled(Boolean(checked));
+ markChanged();
+ }}
+ aria-label={`${voiceEnabled ? "Disable" : "Enable"} voice calls for ${bot.name}`}
+ />
+ </div>
+
<div className="space-y-2">
<div className="text-sm font-medium">Tools</div>
<button
@@ -305,6 +321,7 @@ function BotProfileEditor({
description: normalizedDescription,
engine: nextEngine,
sandbox: nextSandbox,
+ voiceEnabled,
disabledMcpServerIds,
}).then((success) => {
setSaving(false);
diff --git a/apps/web/src/components/roster/BotThreadLanding.tsx b/apps/web/src/components/roster/BotThreadLanding.tsx
index a3e8b2384..01843a489 100644
--- a/apps/web/src/components/roster/BotThreadLanding.tsx
+++ b/apps/web/src/components/roster/BotThreadLanding.tsx
@@ -27,6 +27,7 @@ import { BotConversationScrollArea } from "./BotConversationScrollArea";
import { visibleBotChatMessages } from "./botConversationPresentation";
import { resolveStickyBotEngine } from "./botEngineSelection";
import { BotPromptComposer } from "./BotPromptComposer";
+import { BotVoiceCallButton } from "../voice/VoiceCall";
import { useBotPresence } from "./botPresence";
import { useRosterStore } from "./rosterStore";
import { useBotThreadRuntime } from "./useBotThreadRuntime";
@@ -111,6 +112,9 @@ export function BotThreadLanding({ botId }: { readonly botId: string }) {
<BotAvatarView avatar={bot.avatar} name={bot.name} className="size-6" />
<span className="truncate text-sm font-medium">{bot.name}</span>
</div>
+ <div data-chat-header-actions className="ml-auto flex items-center">
+ <BotVoiceCallButton bot={bot} />
+ </div>
</WorkspacePageHeader>
<BotConversationScrollArea>
{messages.length === 0 ? (
diff --git a/apps/web/src/components/roster/botThreadRuntime.logic.test.ts b/apps/web/src/components/roster/botThreadRuntime.logic.test.ts
index 41fdec565..588a3636e 100644
--- a/apps/web/src/components/roster/botThreadRuntime.logic.test.ts
+++ b/apps/web/src/components/roster/botThreadRuntime.logic.test.ts
@@ -6,9 +6,31 @@ import {
buildGroupTurnStartInput,
findLatestBotThreadTarget,
findLatestGroupThreadTarget,
+ joinOrStartThreadCreate,
} from "./botThreadRuntime.logic";
describe("bot thread runtime", () => {
+ it("shares concurrent initial thread creation", async () => {
+ let retained: { threadId: string } | null = null;
+ const inFlight = { current: null as Promise<{ threadId: string } | null> | null };
+ let starts = 0;
+ const start = async () => {
+ starts += 1;
+ await Promise.resolve();
+ retained = { threadId: "thread-akeru" };
+ return retained;
+ };
+ const join = () => joinOrStartThreadCreate({ getRetained: () => retained, inFlight, start });
+
+ const [first, second] = await Promise.all([join(), join()]);
+ expect(starts).toBe(1);
+ expect(first).toBe(retained);
+ expect(second).toBe(retained);
+
+ expect(await join()).toBe(retained);
+ expect(starts).toBe(1);
+ });
+
it("associates the first durable thread with its bot", () => {
const input = buildBotTurnStartInput({
botId: BotId.make("bot-akeru"),
diff --git a/apps/web/src/components/roster/botThreadRuntime.logic.ts b/apps/web/src/components/roster/botThreadRuntime.logic.ts
index b8e4f458c..0fd4312be 100644
--- a/apps/web/src/components/roster/botThreadRuntime.logic.ts
+++ b/apps/web/src/components/roster/botThreadRuntime.logic.ts
@@ -9,6 +9,22 @@ import type {
ThreadId,
} from "@t3tools/contracts";
+export async function joinOrStartThreadCreate<T>(input: {
+ getRetained: () => T | null;
+ inFlight: { current: Promise<T | null> | null };
+ start: () => Promise<T | null>;
+}): Promise<T | null> {
+ const existing = input.getRetained();
+ if (existing) return existing;
+ const pending = (input.inFlight.current ??= input.start());
+ try {
+ const created = await pending;
+ return input.getRetained() ?? created;
+ } finally {
+ if (input.inFlight.current === pending) input.inFlight.current = null;
+ }
+}
+
export function buildBotTurnStartInput(input: {
botId: BotId;
threadId: ThreadId;
diff --git a/apps/web/src/components/roster/roster.logic.test.ts b/apps/web/src/components/roster/roster.logic.test.ts
index 80f705ffd..608eb841e 100644
--- a/apps/web/src/components/roster/roster.logic.test.ts
+++ b/apps/web/src/components/roster/roster.logic.test.ts
@@ -36,6 +36,7 @@ function bot(input: Partial<Bot> & Pick<Bot, "id" | "name">): Bot {
sandbox: null,
runtimeMode: "full-access",
usageCap: null,
+ voiceEnabled: false,
groupId: null,
pinned: false,
archivedAt: null,
diff --git a/apps/web/src/components/roster/rosterStore.test.ts b/apps/web/src/components/roster/rosterStore.test.ts
index 37a93cda2..bb5b5de36 100644
--- a/apps/web/src/components/roster/rosterStore.test.ts
+++ b/apps/web/src/components/roster/rosterStore.test.ts
@@ -16,6 +16,7 @@ function bot(id: string, archivedAt: string | null = null): Bot {
sandbox: null,
runtimeMode: "full-access",
usageCap: null,
+ voiceEnabled: false,
groupId: null,
pinned: false,
archivedAt,
diff --git a/apps/web/src/components/roster/types.ts b/apps/web/src/components/roster/types.ts
index 2e66653cd..19eb9c3e9 100644
--- a/apps/web/src/components/roster/types.ts
+++ b/apps/web/src/components/roster/types.ts
@@ -34,6 +34,7 @@ export interface Bot {
sandbox: "local" | "vercel" | "akeru-cloud" | "upstash" | null;
runtimeMode: "approval-required" | "auto-accept-edits" | "auto" | "full-access";
usageCap: { unit: "tokens"; limit: number } | null;
+ voiceEnabled: boolean;
groupId: string | null;
pinned: boolean;
archivedAt: string | null;
diff --git a/apps/web/src/components/roster/useBotThreadRuntime.ts b/apps/web/src/components/roster/useBotThreadRuntime.ts
index 6eb043635..734df4e79 100644
--- a/apps/web/src/components/roster/useBotThreadRuntime.ts
+++ b/apps/web/src/components/roster/useBotThreadRuntime.ts
@@ -28,7 +28,11 @@ import { primaryServerProvidersAtom } from "../../state/server";
import { threadEnvironment } from "../../state/threads";
import { useAtomCommand } from "../../state/use-atom-command";
import { sortScopedProjectsForSidebar } from "../Sidebar.logic";
-import { buildBotTurnStartInput, findLatestBotThreadTarget } from "./botThreadRuntime.logic";
+import {
+ buildBotTurnStartInput,
+ findLatestBotThreadTarget,
+ joinOrStartThreadCreate,
+} from "./botThreadRuntime.logic";
import { parseChatPath } from "./roster.logic";
import { useRosterStore } from "./rosterStore";
@@ -131,11 +135,63 @@ export function useBotThreadRuntime(botId: string, effectiveModelSelection: Mode
[providers, settings],
);
const startTurn = useAtomCommand(threadEnvironment.startTurn, { reportFailure: false });
+ const appendVoiceTranscript = useAtomCommand(threadEnvironment.appendVoiceTranscript, {
+ reportFailure: false,
+ });
+ const createThread = useAtomCommand(threadEnvironment.create, { reportFailure: false });
+ const ensureThreadRef = useRef<Promise<ScopedThreadRef | null> | null>(null);
const botReady = serverBots.some((candidate) => candidate.id === botId);
const sendInFlightRef = useRef(false);
const [sending, setSending] = useState(false);
const [error, setError] = useState<string | null>(null);
+ const ensureTranscriptThread = useCallback(
+ async (title = `Call with ${bot?.name ?? "bot"}`): Promise<ScopedThreadRef | null> => {
+ if (!activeProject || !botReady) return null;
+ return joinOrStartThreadCreate({
+ getRetained: () => retainedThreadRef.current.threadRef,
+ inFlight: ensureThreadRef,
+ start: async () => {
+ const threadId = newThreadId();
+ const result = await createThread({
+ environmentId: activeProject.environmentId,
+ input: {
+ threadId,
+ projectId: activeProject.id,
+ botId: BotId.make(botId),
+ title,
+ modelSelection:
+ effectiveModelSelection ??
+ activeProject.defaultModelSelection ??
+ appDefaultModelSelection,
+ runtimeMode: bot?.runtimeMode ?? DEFAULT_RUNTIME_MODE,
+ interactionMode: DEFAULT_INTERACTION_MODE,
+ branch: null,
+ worktreePath: null,
+ createdAt: new Date().toISOString(),
+ },
+ });
+ if (result._tag === "Failure") return null;
+ const threadRef = scopeThreadRef(activeProject.environmentId, threadId);
+ retainedThreadRef.current.threadRef = threadRef;
+ useRosterStore
+ .getState()
+ .recordChatPath(botId, `/${threadRef.environmentId}/${threadRef.threadId}`);
+ return threadRef;
+ },
+ });
+ },
+ [
+ activeProject,
+ appDefaultModelSelection,
+ bot,
+ botId,
+ botReady,
+ createThread,
+ effectiveModelSelection,
+ ],
+ );
+
const send = useCallback(
async (prompt: string, files: readonly File[]): Promise<boolean> => {
if (sendInFlightRef.current) return false;
@@ -157,8 +213,6 @@ export function useBotThreadRuntime(botId: string, effectiveModelSelection: Mode
setSending(true);
setError(null);
const createdAt = new Date().toISOString();
- const currentThreadRef = retainedThreadRef.current.threadRef;
- const threadId = currentThreadRef?.threadId ?? newThreadId();
const modelSelection: ModelSelection =
effectiveModelSelection ?? activeProject.defaultModelSelection ?? appDefaultModelSelection;
const runtimeMode = bot?.runtimeMode ?? DEFAULT_RUNTIME_MODE;
@@ -174,12 +228,17 @@ export function useBotThreadRuntime(botId: string, effectiveModelSelection: Mode
dataUrl: await readFileAsDataUrl(file),
})),
);
- const environmentId = currentThreadRef?.environmentId ?? activeProject.environmentId;
+ const currentThreadRef =
+ retainedThreadRef.current.threadRef ?? (await ensureTranscriptThread(title));
+ if (!currentThreadRef) {
+ setError("Could not send the message.");
+ return false;
+ }
const startResult = await startTurn({
- environmentId,
+ environmentId: currentThreadRef.environmentId,
input: buildBotTurnStartInput({
botId: BotId.make(botId),
- threadId,
+ threadId: currentThreadRef.threadId,
projectId: activeProject.id,
title,
message: {
@@ -192,7 +251,7 @@ export function useBotThreadRuntime(botId: string, effectiveModelSelection: Mode
runtimeMode,
interactionMode: DEFAULT_INTERACTION_MODE,
createdAt,
- createThread: currentThreadRef === null,
+ createThread: false,
}),
});
if (startResult._tag === "Failure") {
@@ -200,11 +259,10 @@ export function useBotThreadRuntime(botId: string, effectiveModelSelection: Mode
return false;
}
- const threadRef = scopeThreadRef(environmentId, threadId);
- retainedThreadRef.current.threadRef = threadRef;
+ retainedThreadRef.current.threadRef = currentThreadRef;
useRosterStore
.getState()
- .recordChatPath(botId, `/${threadRef.environmentId}/${threadRef.threadId}`);
+ .recordChatPath(botId, `/${currentThreadRef.environmentId}/${currentThreadRef.threadId}`);
useRosterStore.getState().recordLastMessage(botId, {
text: prompt || (files.length === 1 ? "Sent an image" : "Sent images"),
at: createdAt,
@@ -225,11 +283,33 @@ export function useBotThreadRuntime(botId: string, effectiveModelSelection: Mode
botId,
effectiveModelSelection,
botReady,
+ ensureTranscriptThread,
startTurn,
],
);
+ const appendTranscript = useCallback(
+ async (role: "user" | "assistant", text: string) => {
+ const trimmed = text.trim();
+ if (trimmed.length === 0) return;
+ const threadRef = retainedThreadRef.current.threadRef ?? (await ensureTranscriptThread());
+ if (!threadRef) return;
+ void appendVoiceTranscript({
+ environmentId: threadRef.environmentId,
+ input: {
+ threadId: threadRef.threadId,
+ messageId: newMessageId(),
+ role,
+ text: trimmed,
+ ...(role === "assistant" ? { respondingBotId: BotId.make(botId) } : {}),
+ },
+ });
+ },
+ [appendVoiceTranscript, botId, ensureTranscriptThread],
+ );
+
return {
+ appendTranscript,
bootstrapped,
botReady,
defaultProject: activeProject,
diff --git a/apps/web/src/components/settings/SettingsDialog.tsx b/apps/web/src/components/settings/SettingsDialog.tsx
index 31480c50b..56d50a93a 100644
--- a/apps/web/src/components/settings/SettingsDialog.tsx
+++ b/apps/web/src/components/settings/SettingsDialog.tsx
@@ -1,5 +1,6 @@
import {
BotIcon,
+ CallIcon,
KeyboardIcon,
Link02Icon,
PaintBrush01Icon,
@@ -23,6 +24,9 @@ const AppearanceSettingsPanel = lazy(async () => ({
const ProvidersPanel = lazy(async () => ({
default: (await import("./ProvidersPanel")).ProvidersPanel,
}));
+const VoiceSettingsPanel = lazy(async () => ({
+ default: (await import("./VoiceSettings")).VoiceSettingsPanel,
+}));
const ConnectionsSettings = lazy(async () => ({
default: (await import("./ConnectionsSettings")).ConnectionsSettings,
}));
@@ -40,6 +44,7 @@ const SECTION_PANELS: Readonly<Record<SettingsSection, ComponentType>> = {
general: GeneralSettingsPanel,
appearance: AppearanceSettingsPanel,
providers: ProvidersPanel,
+ voice: VoiceSettingsPanel,
connections: ConnectionsSettings,
keybindings: KeybindingsSettingsPanel,
"source-control": SourceControlSettingsPanel,
@@ -55,6 +60,7 @@ const NAV_ITEMS: ReadonlyArray<{
{ section: "general", label: "General", icon: Settings02Icon },
{ section: "appearance", label: "Appearance", icon: PaintBrush01Icon },
{ section: "providers", label: "Providers", icon: BotIcon },
+ { section: "voice", label: "Voice", icon: CallIcon },
{ section: "connections", label: "Connections", icon: Link02Icon },
{ section: "keybindings", label: "Keybindings", icon: KeyboardIcon },
];
diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx
index 068dfe2f6..194ab50d1 100644
--- a/apps/web/src/components/settings/SettingsPanels.tsx
+++ b/apps/web/src/components/settings/SettingsPanels.tsx
@@ -546,6 +546,7 @@ export function useSettingsRestore(onRestored?: () => void) {
...(settings.enableAgentBrowserAccess !== DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess
? ["Agent browser access"]
: []),
+ ...(!Equal.equals(settings.voice, DEFAULT_UNIFIED_SETTINGS.voice) ? ["Voice"] : []),
],
[
isTextGenerationModelDirty,
@@ -556,6 +557,7 @@ export function useSettingsRestore(onRestored?: () => void) {
settings.browserAutoShowFloatingPreview,
settings.appearanceContrast,
settings.enableAgentBrowserAccess,
+ settings.voice,
settings.confirmQuit,
settings.confirmThreadArchive,
settings.confirmThreadDelete,
@@ -693,6 +695,7 @@ export function useSettingsRestore(onRestored?: () => void) {
// name, so a user restoring defaults is told the agent regains access
// rather than discovering it later.
enableAgentBrowserAccess: DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess,
+ voice: DEFAULT_UNIFIED_SETTINGS.voice,
});
onRestored?.();
}, [
diff --git a/apps/web/src/components/settings/VoiceSettings.tsx b/apps/web/src/components/settings/VoiceSettings.tsx
new file mode 100644
index 000000000..ec4e01097
--- /dev/null
+++ b/apps/web/src/components/settings/VoiceSettings.tsx
@@ -0,0 +1,117 @@
+import {
+ CHATGPT_REALTIME_VOICES,
+ DEFAULT_SERVER_SETTINGS,
+ type ChatGptRealtimeVoice,
+} from "@t3tools/contracts";
+
+import { usePrimarySettings, useUpdatePrimarySettings } from "~/hooks/useSettings";
+import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
+import { Switch } from "../ui/switch";
+import {
+ SettingResetButton,
+ SettingsPageContainer,
+ SettingsRow,
+ SettingsSection,
+} from "./settingsLayout";
+import { searchableSetting } from "./settingsSearch";
+
+const VOICE_LABELS: Readonly<Record<ChatGptRealtimeVoice, string>> = {
+ alloy: "Alloy",
+ ash: "Ash",
+ ballad: "Ballad",
+ coral: "Coral",
+ echo: "Echo",
+ sage: "Sage",
+ shimmer: "Shimmer",
+ verse: "Verse",
+ marin: "Marin",
+ cedar: "Cedar",
+};
+
+export function VoiceSettingsPanel() {
+ const voice = usePrimarySettings((settings) => settings.voice);
+ const updateSettings = useUpdatePrimarySettings();
+
+ const selectVoice = (value: string | null) => {
+ const selected = CHATGPT_REALTIME_VOICES.find((candidate) => candidate === value);
+ if (selected) updateSettings({ voice: { voice: selected } });
+ };
+
+ return (
+ <SettingsPageContainer>
+ <SettingsSection id="voice" title="Voice">
+ <SettingsRow
+ {...searchableSetting("voice-enabled")}
+ description="Allow bots with Voice calls enabled to start subscription voice calls."
+ resetAction={
+ voice.enabled !== DEFAULT_SERVER_SETTINGS.voice.enabled ? (
+ <SettingResetButton
+ label="voice"
+ onClick={() =>
+ updateSettings({ voice: { enabled: DEFAULT_SERVER_SETTINGS.voice.enabled } })
+ }
+ />
+ ) : null
+ }
+ control={
+ <Switch
+ checked={voice.enabled}
+ onCheckedChange={(checked) =>
+ updateSettings({ voice: { enabled: Boolean(checked) } })
+ }
+ aria-label="Enable voice"
+ />
+ }
+ />
+ <SettingsRow
+ {...searchableSetting("voice-provider")}
+ description="Use the ChatGPT subscription connected to this environment."
+ control={
+ <Select
+ value={voice.provider}
+ onValueChange={(provider) => {
+ if (provider === "chatgpt") updateSettings({ voice: { provider } });
+ }}
+ disabled={!voice.enabled}
+ >
+ <SelectTrigger className="w-full sm:w-52" aria-label="Voice provider">
+ <SelectValue>ChatGPT subscription</SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ <SelectItem value="chatgpt">ChatGPT subscription</SelectItem>
+ </SelectPopup>
+ </Select>
+ }
+ />
+ <SettingsRow
+ {...searchableSetting("voice-selection")}
+ description="Choose the voice used for new calls."
+ resetAction={
+ voice.enabled && voice.voice !== DEFAULT_SERVER_SETTINGS.voice.voice ? (
+ <SettingResetButton
+ label="voice selection"
+ onClick={() =>
+ updateSettings({ voice: { voice: DEFAULT_SERVER_SETTINGS.voice.voice } })
+ }
+ />
+ ) : null
+ }
+ control={
+ <Select value={voice.voice} onValueChange={selectVoice} disabled={!voice.enabled}>
+ <SelectTrigger className="w-full sm:w-52" aria-label="Voice">
+ <SelectValue>{VOICE_LABELS[voice.voice]}</SelectValue>
+ </SelectTrigger>
+ <SelectPopup align="end" alignItemWithTrigger={false}>
+ {CHATGPT_REALTIME_VOICES.map((candidate) => (
+ <SelectItem key={candidate} value={candidate}>
+ {VOICE_LABELS[candidate]}
+ </SelectItem>
+ ))}
+ </SelectPopup>
+ </Select>
+ }
+ />
+ </SettingsSection>
+ </SettingsPageContainer>
+ );
+}
diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts
index 2c813f401..b05e31568 100644
--- a/apps/web/src/components/settings/settingsSearch.ts
+++ b/apps/web/src/components/settings/settingsSearch.ts
@@ -5,6 +5,7 @@ export type SettingsPath =
| "/settings/appearance"
| "/settings/keybindings"
| "/settings/providers"
+ | "/settings/voice"
| "/settings/integrations"
| "/settings/source-control"
| "/settings/connections"
@@ -29,6 +30,7 @@ export const SETTINGS_SECTION_LABELS: Readonly<Record<SettingsPath, string>> = {
"/settings/appearance": "Appearance",
"/settings/keybindings": "Keybindings",
"/settings/providers": "Providers",
+ "/settings/voice": "Voice",
"/settings/integrations": "Integrations",
"/settings/source-control": "Source Control",
"/settings/connections": "Connections",
@@ -146,6 +148,21 @@ export const SETTINGS_SEARCH_ITEMS = [
title: "Provider update checks",
to: "/settings/general",
},
+ {
+ id: "voice-enabled",
+ title: "Voice",
+ to: "/settings/voice",
+ },
+ {
+ id: "voice-provider",
+ title: "Voice provider",
+ to: "/settings/voice",
+ },
+ {
+ id: "voice-selection",
+ title: "Voice selection",
+ to: "/settings/voice",
+ },
{
id: "new-threads",
title: "New threads",
diff --git a/apps/web/src/components/voice/VoiceCall.test.tsx b/apps/web/src/components/voice/VoiceCall.test.tsx
new file mode 100644
index 000000000..3c9e66192
--- /dev/null
+++ b/apps/web/src/components/voice/VoiceCall.test.tsx
@@ -0,0 +1,248 @@
+import { BotId } from "@t3tools/contracts";
+import { renderToStaticMarkup } from "react-dom/server";
+import { describe, expect, it, vi } from "vite-plus/test";
+
+import type { Bot } from "../roster/types";
+import {
+ BotVoiceCallButtonView,
+ handleVoiceChannelMessage,
+ reduceVoiceCallUiState,
+ resolveVoiceCallOfferSdp,
+ scheduleVoiceDisconnectTimeout,
+ voiceConnectionStateAction,
+ waitForIceGathering,
+ type VoiceCallUiState,
+} from "./VoiceCall";
+
+const bot: Bot = {
+ id: "bot-akeru",
+ name: "Akeru",
+ title: "Generalist",
+ label: null,
+ description: null,
+ disabledMcpServerIds: [],
+ avatar: { kind: "blob", shape: "circle", color: "#5B7FD4" },
+ engine: null,
+ sandbox: null,
+ runtimeMode: "full-access",
+ usageCap: null,
+ voiceEnabled: false,
+ groupId: null,
+ pinned: false,
+ archivedAt: null,
+ createdAt: "2026-08-27T00:00:00.000Z",
+ updatedAt: "2026-08-27T00:00:00.000Z",
+};
+
+describe("voice channel messages", () => {
+ const makeHandlers = (delivered = true) => ({
+ appendTranscript: vi.fn<(role: "user" | "assistant", text: string) => void>(),
+ sendGoalMessage: vi.fn<(text: string) => Promise<boolean>>(async () => delivered),
+ });
+ const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
+
+ it("appends both sides of the conversation to the chat", () => {
+ const handlers = makeHandlers();
+ const reply = vi.fn<(payload: string) => void>();
+ handleVoiceChannelMessage(
+ JSON.stringify({
+ type: "conversation.item.input_audio_transcription.completed",
+ transcript: " Ship the fix today. ",
+ }),
+ handlers,
+ reply,
+ );
+ handleVoiceChannelMessage(
+ JSON.stringify({
+ type: "response.output_audio_transcript.done",
+ transcript: "On it.",
+ }),
+ handlers,
+ reply,
+ );
+ expect(handlers.appendTranscript.mock.calls).toEqual([
+ ["user", "Ship the fix today."],
+ ["assistant", "On it."],
+ ]);
+ expect(handlers.sendGoalMessage).not.toHaveBeenCalled();
+ expect(reply).not.toHaveBeenCalled();
+ });
+
+ it("forwards a send_to_chat call as a goal message and confirms to the model", async () => {
+ const handlers = makeHandlers();
+ const reply = vi.fn<(payload: string) => void>();
+ handleVoiceChannelMessage(
+ JSON.stringify({
+ type: "response.function_call_arguments.done",
+ name: "send_to_chat",
+ call_id: "call-abc",
+ arguments: JSON.stringify({ message: "Add retries to the sync job." }),
+ }),
+ handlers,
+ reply,
+ );
+ await flush();
+ expect(handlers.sendGoalMessage).toHaveBeenCalledWith("Add retries to the sync job.");
+ expect(handlers.appendTranscript).not.toHaveBeenCalled();
+ const output = JSON.parse(reply.mock.calls[0]![0]) as { item: { output: string } };
+ const respond = JSON.parse(reply.mock.calls[1]![0]) as unknown;
+ expect(output).toMatchObject({
+ type: "conversation.item.create",
+ item: { type: "function_call_output", call_id: "call-abc" },
+ });
+ expect(JSON.parse(output.item.output)).toEqual({ ok: true, delivered: "chat" });
+ expect(respond).toMatchObject({ type: "response.create" });
+ });
+
+ it("reports a failed delivery to the model instead of claiming success", async () => {
+ const handlers = makeHandlers(false);
+ const reply = vi.fn<(payload: string) => void>();
+ handleVoiceChannelMessage(
+ JSON.stringify({
+ type: "response.function_call_arguments.done",
+ name: "send_to_chat",
+ call_id: "call-def",
+ arguments: JSON.stringify({ message: "Fix the flaky test." }),
+ }),
+ handlers,
+ reply,
+ );
+ await flush();
+ const output = JSON.parse(reply.mock.calls[0]![0]) as { item: { output: string } };
+ expect(JSON.parse(output.item.output)).toMatchObject({ ok: false });
+ });
+
+ it("acknowledges malformed tool arguments instead of hanging the call", async () => {
+ const handlers = makeHandlers();
+ const reply = vi.fn<(payload: string) => void>();
+ handleVoiceChannelMessage("not json", handlers, reply);
+ expect(reply).not.toHaveBeenCalled();
+ handleVoiceChannelMessage(
+ JSON.stringify({
+ type: "response.function_call_arguments.done",
+ name: "send_to_chat",
+ call_id: "call-broken",
+ arguments: "broken",
+ }),
+ handlers,
+ reply,
+ );
+ await flush();
+ expect(handlers.sendGoalMessage).not.toHaveBeenCalled();
+ const output = JSON.parse(reply.mock.calls[0]![0]) as { item: { output: string } };
+ expect(JSON.parse(output.item.output)).toMatchObject({ ok: false });
+ });
+});
+
+describe("voice call UI", () => {
+ it("hides the phone button when voice is off and shows it when voice is on", () => {
+ expect(
+ renderToStaticMarkup(
+ <BotVoiceCallButtonView
+ bot={bot}
+ active={false}
+ disabled={false}
+ globallyEnabled
+ onClick={() => {}}
+ />,
+ ),
+ ).toBe("");
+
+ const enabled = renderToStaticMarkup(
+ <BotVoiceCallButtonView
+ bot={{ ...bot, voiceEnabled: true }}
+ active={false}
+ disabled={false}
+ globallyEnabled
+ onClick={() => {}}
+ />,
+ );
+ expect(enabled).toContain('aria-label="Call Akeru"');
+
+ expect(
+ renderToStaticMarkup(
+ <BotVoiceCallButtonView
+ bot={{ ...bot, voiceEnabled: true }}
+ active={false}
+ disabled={false}
+ globallyEnabled={false}
+ onClick={() => {}}
+ />,
+ ),
+ ).toBe("");
+ });
+
+ it("uses the complete offer while ICE gathering remains in progress", () => {
+ expect(
+ resolveVoiceCallOfferSdp(
+ { iceGatheringState: "gathering", localDescription: null },
+ { type: "offer", sdp: "complete-offer" },
+ ),
+ ).toBe("complete-offer");
+ });
+
+ it("continues when ICE gathering does not report completion", async () => {
+ vi.useFakeTimers();
+ try {
+ const peer = {
+ iceGatheringState: "gathering",
+ addEventListener: vi.fn(),
+ removeEventListener: vi.fn(),
+ } as unknown as RTCPeerConnection;
+ const gathered = waitForIceGathering(peer, 100);
+ vi.advanceTimersByTime(100);
+ await gathered;
+ expect(peer.removeEventListener).toHaveBeenCalledWith(
+ "icegatheringstatechange",
+ expect.any(Function),
+ );
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("keeps the recovery window until a disconnected peer reconnects", () => {
+ expect(voiceConnectionStateAction("disconnected")).toBe("wait-for-recovery");
+ expect(voiceConnectionStateAction("connecting")).toBe("keep-recovery-window");
+ expect(voiceConnectionStateAction("new")).toBe("keep-recovery-window");
+ expect(voiceConnectionStateAction("connected")).toBe("recovered");
+ expect(voiceConnectionStateAction("failed")).toBe("end");
+ expect(voiceConnectionStateAction("closed")).toBe("end");
+ });
+
+ it("ends the recovery window unless the peer reaches connected", () => {
+ vi.useFakeTimers();
+ try {
+ let state: RTCPeerConnectionState = "disconnected";
+ const results: boolean[] = [];
+ scheduleVoiceDisconnectTimeout(
+ () => state,
+ (recovered) => results.push(recovered),
+ );
+ state = "connecting";
+ vi.advanceTimersByTime(5_000);
+ expect(results).toEqual([false]);
+
+ scheduleVoiceDisconnectTimeout(
+ () => state,
+ (recovered) => results.push(recovered),
+ );
+ state = "connected";
+ vi.advanceTimersByTime(5_000);
+ expect(results).toEqual([false, true]);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("clears the persistent call state on hangup", () => {
+ const active: VoiceCallUiState = {
+ callId: "call-1",
+ status: "live",
+ botId: BotId.make("bot-akeru"),
+ botName: "Akeru",
+ startedAt: "2026-08-27T00:00:00.000Z",
+ };
+ expect(reduceVoiceCallUiState(active, { type: "hung-up" })).toBeNull();
+ });
+});
diff --git a/apps/web/src/components/voice/VoiceCall.tsx b/apps/web/src/components/voice/VoiceCall.tsx
new file mode 100644
index 000000000..a099db5c1
--- /dev/null
+++ b/apps/web/src/components/voice/VoiceCall.tsx
@@ -0,0 +1,486 @@
+import { CallEndIcon, CallIcon } from "@hugeicons/core-free-icons";
+import { BotId, type VoiceCallSnapshot } from "@t3tools/contracts";
+import { useNavigate } from "@tanstack/react-router";
+import * as Cause from "effect/Cause";
+import {
+ createContext,
+ useCallback,
+ useContext,
+ useEffect,
+ useMemo,
+ useReducer,
+ useRef,
+ useState,
+ type ReactNode,
+} from "react";
+
+import type { Bot } from "../roster/types";
+import { useBotThreadRuntime } from "../roster/useBotThreadRuntime";
+import { useRosterStore } from "../roster/rosterStore";
+import { Button } from "../ui/button";
+import { AppIcon } from "../ui/app-icon";
+import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
+import { toastManager } from "../ui/toast";
+import { usePrimarySettings } from "../../hooks/useSettings";
+import { serverEnvironment } from "../../state/server";
+import { usePrimaryEnvironmentId } from "../../state/environments";
+import { useAtomCommand } from "../../state/use-atom-command";
+
+interface ActiveBrowserCall {
+ readonly call: Exclude<VoiceCallSnapshot, { status: "idle" }>;
+ readonly environmentId: NonNullable<ReturnType<typeof usePrimaryEnvironmentId>>;
+ readonly peer: RTCPeerConnection;
+ readonly microphone: MediaStream;
+ readonly speaker: HTMLAudioElement;
+ disconnectTimer: ReturnType<typeof setTimeout> | null;
+}
+
+export type VoiceCallUiState = ActiveBrowserCall["call"] | null;
+export type VoiceCallUiAction =
+ | { readonly type: "connected"; readonly call: ActiveBrowserCall["call"] }
+ | { readonly type: "hung-up" };
+
+export function reduceVoiceCallUiState(
+ state: VoiceCallUiState,
+ action: VoiceCallUiAction,
+): VoiceCallUiState {
+ return action.type === "connected" ? action.call : null;
+}
+
+export interface VoiceCallChatHandlers {
+ readonly appendTranscript: (role: "user" | "assistant", text: string) => void | Promise<void>;
+ readonly sendGoalMessage: (text: string) => boolean | Promise<boolean>;
+}
+
+function stringField(value: unknown, field: string): string | null {
+ if (typeof value !== "object" || value === null) return null;
+ const candidate = (value as Record<string, unknown>)[field];
+ return typeof candidate === "string" ? candidate : null;
+}
+
+export function handleVoiceChannelMessage(
+ raw: string,
+ handlers: VoiceCallChatHandlers,
+ reply: (payload: string) => void,
+): void {
+ let event: unknown;
+ try {
+ event = JSON.parse(raw);
+ } catch {
+ return;
+ }
+ const type = stringField(event, "type");
+ const transcript = stringField(event, "transcript")?.trim() ?? "";
+ if (type === "conversation.item.input_audio_transcription.completed" && transcript.length > 0) {
+ handlers.appendTranscript("user", transcript);
+ return;
+ }
+ if (
+ (type === "response.output_audio_transcript.done" ||
+ type === "response.audio_transcript.done") &&
+ transcript.length > 0
+ ) {
+ handlers.appendTranscript("assistant", transcript);
+ return;
+ }
+ if (
+ type === "response.function_call_arguments.done" &&
+ stringField(event, "name") === "send_to_chat"
+ ) {
+ const callId = stringField(event, "call_id");
+ const finish = (output: Record<string, unknown>) => {
+ if (callId === null) return;
+ reply(
+ JSON.stringify({
+ type: "conversation.item.create",
+ item: {
+ type: "function_call_output",
+ call_id: callId,
+ output: JSON.stringify(output),
+ },
+ }),
+ );
+ reply(JSON.stringify({ type: "response.create" }));
+ };
+ let parsedArguments: unknown;
+ try {
+ parsedArguments = JSON.parse(stringField(event, "arguments") ?? "");
+ } catch {
+ finish({ ok: false, error: "Could not parse the tool arguments." });
+ return;
+ }
+ const message = stringField(parsedArguments, "message")?.trim() ?? "";
+ if (message.length === 0) {
+ finish({ ok: false, error: "The message was empty." });
+ return;
+ }
+ void Promise.resolve(handlers.sendGoalMessage(message))
+ .catch(() => false)
+ .then((delivered) =>
+ finish(
+ delivered
+ ? { ok: true, delivered: "chat" }
+ : { ok: false, error: "The chat did not accept the message." },
+ ),
+ );
+ }
+}
+
+interface VoiceCallContextValue {
+ readonly activeCall: ActiveBrowserCall["call"] | null;
+ readonly startingBotId: string | null;
+ readonly startOrReturn: (bot: Bot, chatHandlers?: VoiceCallChatHandlers) => void;
+ readonly hangup: () => void;
+ readonly returnToCall: () => void;
+}
+
+const VoiceCallContext = createContext<VoiceCallContextValue | null>(null);
+
+export function waitForIceGathering(peer: RTCPeerConnection, timeoutMs = 5_000): Promise<void> {
+ if (peer.iceGatheringState === "complete") return Promise.resolve();
+ return new Promise((resolve) => {
+ const finish = () => {
+ clearTimeout(timer);
+ peer.removeEventListener("icegatheringstatechange", onChange);
+ resolve();
+ };
+ const onChange = () => {
+ if (peer.iceGatheringState === "complete") finish();
+ };
+ const timer = setTimeout(finish, timeoutMs);
+ peer.addEventListener("icegatheringstatechange", onChange);
+ });
+}
+
+export function resolveVoiceCallOfferSdp(
+ peer: Pick<RTCPeerConnection, "iceGatheringState" | "localDescription">,
+ offer: RTCSessionDescriptionInit,
+): string | undefined {
+ return peer.iceGatheringState === "complete"
+ ? (peer.localDescription?.sdp ?? offer.sdp)
+ : offer.sdp;
+}
+
+export function voiceConnectionStateAction(
+ state: RTCPeerConnectionState,
+): "end" | "recovered" | "wait-for-recovery" | "keep-recovery-window" {
+ if (state === "failed" || state === "closed") return "end";
+ if (state === "connected") return "recovered";
+ if (state === "disconnected") return "wait-for-recovery";
+ return "keep-recovery-window";
+}
+
+export function scheduleVoiceDisconnectTimeout(
+ getConnectionState: () => RTCPeerConnectionState,
+ onTimeout: (recovered: boolean) => void,
+): ReturnType<typeof setTimeout> {
+ return setTimeout(() => onTimeout(getConnectionState() === "connected"), 5_000);
+}
+
+function stopBrowserCall(active: ActiveBrowserCall): void {
+ if (active.disconnectTimer !== null) clearTimeout(active.disconnectTimer);
+ active.microphone.getTracks().forEach((track) => track.stop());
+ active.peer.close();
+ active.speaker.pause();
+ active.speaker.srcObject = null;
+}
+
+export function VoiceCallProvider({ children }: { readonly children: ReactNode }) {
+ const navigate = useNavigate();
+ const environmentId = usePrimaryEnvironmentId();
+ const startVoiceCall = useAtomCommand(serverEnvironment.startVoiceCall, { reportFailure: false });
+ const hangupVoiceCall = useAtomCommand(serverEnvironment.hangupVoiceCall, {
+ reportFailure: false,
+ });
+ const [activeCall, dispatchCall] = useReducer(reduceVoiceCallUiState, null);
+ const [startingBotId, setStartingBotId] = useState<string | null>(null);
+ const activeRef = useRef<ActiveBrowserCall | null>(null);
+ const startingBotRef = useRef<string | null>(null);
+ const hangupCommandRef = useRef(hangupVoiceCall);
+ hangupCommandRef.current = hangupVoiceCall;
+
+ const returnToCall = useCallback(() => {
+ const call = activeRef.current?.call;
+ if (!call) return;
+ useRosterStore.getState().selectBot(call.botId);
+ void navigate({ to: "/bots/$botId", params: { botId: call.botId } });
+ }, [navigate]);
+
+ const startOrReturn = useCallback(
+ (bot: Bot, chatHandlers?: VoiceCallChatHandlers) => {
+ const current = activeRef.current;
+ if (current !== null) {
+ if (current.call.botId === bot.id) returnToCall();
+ else toastManager.add({ type: "warning", title: "A call is already active" });
+ return;
+ }
+ if (startingBotRef.current !== null || environmentId === null) return;
+ startingBotRef.current = bot.id;
+ setStartingBotId(bot.id);
+ void (async () => {
+ let microphone: MediaStream | null = null;
+ let peer: RTCPeerConnection | null = null;
+ let speaker: HTMLAudioElement | null = null;
+ let serverCallId: string | null = null;
+ try {
+ microphone = await navigator.mediaDevices.getUserMedia({ audio: true });
+ peer = new RTCPeerConnection();
+ speaker = new Audio();
+ speaker.autoplay = true;
+ peer.ontrack = (event) => {
+ if (!event.streams[0]) return;
+ speaker!.srcObject = event.streams[0];
+ void speaker!.play().catch(() => undefined);
+ };
+ microphone.getTracks().forEach((track) => peer!.addTrack(track, microphone!));
+ const events = peer.createDataChannel("oai-events");
+ if (chatHandlers) {
+ events.onmessage = (channelMessage) => {
+ handleVoiceChannelMessage(String(channelMessage.data), chatHandlers, (payload) => {
+ if (events.readyState === "open") events.send(payload);
+ });
+ };
+ }
+ const offer = await peer.createOffer();
+ await peer.setLocalDescription(offer);
+ await waitForIceGathering(peer);
+ const sdp = resolveVoiceCallOfferSdp(peer, offer);
+ if (!sdp) throw new Error("The microphone did not produce a call offer.");
+ const result = await startVoiceCall({
+ environmentId,
+ input: { botId: BotId.make(bot.id), sdp },
+ });
+ if (result._tag === "Failure") {
+ const cause = Cause.squash(result.cause);
+ throw new Error(
+ cause instanceof Error ? cause.message : "Could not start the voice call.",
+ );
+ }
+ serverCallId = result.value.call.callId;
+ await peer.setRemoteDescription({ type: "answer", sdp: result.value.answerSdp });
+ const browserCall: ActiveBrowserCall = {
+ call: result.value.call,
+ environmentId,
+ peer,
+ microphone,
+ speaker,
+ disconnectTimer: null,
+ };
+ activeRef.current = browserCall;
+ dispatchCall({ type: "connected", call: browserCall.call });
+ const endDisconnectedCall = () => {
+ if (activeRef.current !== browserCall) return;
+ activeRef.current = null;
+ stopBrowserCall(browserCall);
+ dispatchCall({ type: "hung-up" });
+ void hangupCommandRef.current({
+ environmentId: browserCall.environmentId,
+ input: { callId: browserCall.call.callId },
+ });
+ toastManager.add({ type: "warning", title: "Call ended" });
+ };
+ browserCall.peer.onconnectionstatechange = () => {
+ const action = voiceConnectionStateAction(browserCall.peer.connectionState);
+ if (action === "recovered") {
+ if (browserCall.disconnectTimer !== null) {
+ clearTimeout(browserCall.disconnectTimer);
+ browserCall.disconnectTimer = null;
+ }
+ return;
+ }
+ if (action === "wait-for-recovery") {
+ browserCall.disconnectTimer ??= scheduleVoiceDisconnectTimeout(
+ () => browserCall.peer.connectionState,
+ (recovered) => {
+ browserCall.disconnectTimer = null;
+ if (!recovered) endDisconnectedCall();
+ },
+ );
+ return;
+ }
+ if (action === "end") endDisconnectedCall();
+ };
+ } catch (error) {
+ microphone?.getTracks().forEach((track) => track.stop());
+ peer?.close();
+ if (speaker) speaker.srcObject = null;
+ if (serverCallId !== null) {
+ void hangupVoiceCall({ environmentId, input: { callId: serverCallId } });
+ }
+ toastManager.add({
+ type: "error",
+ title: "Could not start call",
+ description: error instanceof Error ? error.message : "Check microphone access.",
+ });
+ } finally {
+ if (startingBotRef.current === bot.id) startingBotRef.current = null;
+ setStartingBotId(null);
+ }
+ })();
+ },
+ [environmentId, hangupVoiceCall, returnToCall, startVoiceCall, startingBotId],
+ );
+
+ const hangup = useCallback(() => {
+ const current = activeRef.current;
+ if (!current) return;
+ activeRef.current = null;
+ stopBrowserCall(current);
+ dispatchCall({ type: "hung-up" });
+ void hangupVoiceCall({
+ environmentId: current.environmentId,
+ input: { callId: current.call.callId },
+ }).then((result) => {
+ if (result._tag === "Failure") {
+ toastManager.add({ type: "warning", title: "Call ended locally" });
+ }
+ });
+ }, [hangupVoiceCall]);
+
+ useEffect(
+ () => () => {
+ const current = activeRef.current;
+ if (!current) return;
+ activeRef.current = null;
+ stopBrowserCall(current);
+ void hangupCommandRef.current({
+ environmentId: current.environmentId,
+ input: { callId: current.call.callId },
+ });
+ },
+ [],
+ );
+
+ const value = useMemo<VoiceCallContextValue>(
+ () => ({ activeCall, startingBotId, startOrReturn, hangup, returnToCall }),
+ [activeCall, hangup, returnToCall, startOrReturn, startingBotId],
+ );
+
+ return (
+ <VoiceCallContext.Provider value={value}>
+ {children}
+ <VoiceCallBar />
+ </VoiceCallContext.Provider>
+ );
+}
+
+function useVoiceCall() {
+ const value = useContext(VoiceCallContext);
+ if (!value) throw new Error("Voice call controls must be inside VoiceCallProvider.");
+ return value;
+}
+
+export function BotVoiceCallButtonView({
+ bot,
+ active,
+ disabled,
+ globallyEnabled,
+ onClick,
+}: {
+ readonly bot: Bot;
+ readonly active: boolean;
+ readonly disabled: boolean;
+ readonly globallyEnabled: boolean;
+ readonly onClick: () => void;
+}) {
+ if (!globallyEnabled || !bot.voiceEnabled) return null;
+ const label = active ? `Return to call with ${bot.name}` : `Call ${bot.name}`;
+ return (
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <Button
+ type="button"
+ size="icon-sm"
+ variant="ghost"
+ aria-label={label}
+ disabled={disabled}
+ onClick={onClick}
+ />
+ }
+ >
+ <AppIcon icon={CallIcon} />
+ </TooltipTrigger>
+ <TooltipPopup side="bottom">{label}</TooltipPopup>
+ </Tooltip>
+ );
+}
+
+export function BotVoiceCallButton({ bot }: { readonly bot: Bot }) {
+ const { activeCall, startingBotId, startOrReturn } = useVoiceCall();
+ const globallyEnabled = usePrimarySettings((settings) => settings.voice.enabled);
+ const runtime = useBotThreadRuntime(bot.id, null);
+ const runtimeRef = useRef(runtime);
+ runtimeRef.current = runtime;
+ const chatHandlers = useMemo<VoiceCallChatHandlers>(
+ () => ({
+ appendTranscript: (role, text) => runtimeRef.current.appendTranscript(role, text),
+ sendGoalMessage: (text) => runtimeRef.current.send(text, []),
+ }),
+ [],
+ );
+ return (
+ <BotVoiceCallButtonView
+ bot={bot}
+ active={activeCall?.botId === bot.id}
+ disabled={startingBotId !== null}
+ globallyEnabled={globallyEnabled}
+ onClick={() => startOrReturn(bot, chatHandlers)}
+ />
+ );
+}
+
+export function SelectedBotVoiceCallButton() {
+ const bot = useRosterStore((state) =>
+ state.selectedBotId === null
+ ? null
+ : (state.bots.find((candidate) => candidate.id === state.selectedBotId) ?? null),
+ );
+ return bot ? <BotVoiceCallButton bot={bot} /> : null;
+}
+
+function VoiceCallBar() {
+ const { activeCall, hangup, returnToCall } = useVoiceCall();
+ const [now, setNow] = useState(() => Date.now());
+
+ useEffect(() => {
+ if (!activeCall) return;
+ setNow(Date.now());
+ const timer = window.setInterval(() => setNow(Date.now()), 1_000);
+ return () => window.clearInterval(timer);
+ }, [activeCall]);
+
+ if (!activeCall) return null;
+ const elapsedSeconds = Math.max(0, Math.floor((now - Date.parse(activeCall.startedAt)) / 1_000));
+ const minutes = Math.floor(elapsedSeconds / 60);
+ const seconds = String(elapsedSeconds % 60).padStart(2, "0");
+
+ return (
+ <div className="pointer-events-none fixed inset-x-0 top-2 z-60 flex justify-center px-4">
+ <div className="pointer-events-auto flex h-10 items-center rounded-full border border-border bg-background/95 pl-4 pr-1.5 shadow-lg backdrop-blur">
+ <button
+ type="button"
+ aria-label={`Return to call with ${activeCall.botName}`}
+ className="flex items-center gap-2 pr-3"
+ onClick={returnToCall}
+ >
+ <span className="size-2 rounded-full bg-success" />
+ <span className="text-sm font-medium">{activeCall.botName}</span>
+ <span className="text-xs tabular-nums text-muted-foreground">
+ {minutes}:{seconds}
+ </span>
+ </button>
+ <Button
+ type="button"
+ size="icon-sm"
+ variant="destructive"
+ aria-label="Hang up"
+ className="rounded-full"
+ onClick={hangup}
+ >
+ <AppIcon icon={CallEndIcon} />
+ </Button>
+ </div>
+ </div>
+ );
+}
diff --git a/apps/web/src/routes/__root.tsx b/apps/web/src/routes/__root.tsx
index c2d14d3ba..73d998273 100644
--- a/apps/web/src/routes/__root.tsx
+++ b/apps/web/src/routes/__root.tsx
@@ -13,6 +13,7 @@ import { ConnectOnboardingDialog } from "../components/cloud/ConnectOnboardingDi
import { RelayClientInstallDialog } from "../components/cloud/RelayClientInstallDialog";
import { SshPasswordPromptDialog } from "../components/desktop/SshPasswordPromptDialog";
import { ProviderUpdateLaunchNotification } from "../components/ProviderUpdateLaunchNotification";
+import { VoiceCallProvider } from "../components/voice/VoiceCall";
import { SlowRpcRequestToastCoordinator } from "../components/SlowRpcRequestToastCoordinator";
import { ThemeEditorHost } from "../components/settings/ThemeEditorHost";
import { SettingsDialog } from "../components/settings/SettingsDialog";
@@ -142,7 +143,7 @@ function RootRouteView() {
{primaryEnvironmentAuthenticated ? <EventRouter /> : null}
{primaryEnvironmentAuthenticated ? <PlanAgentSelectionHeal /> : null}
{primaryEnvironmentAuthenticated ? <ProviderUpdateLaunchNotification /> : null}
- {appShell}
+ <VoiceCallProvider>{appShell}</VoiceCallProvider>
<SettingsDialog />
<PluginsDialog />
<UsageDialog />
diff --git a/apps/web/src/routes/_chat.bots.$botId.tsx b/apps/web/src/routes/_chat.bots.$botId.tsx
index 528046ebe..88870c749 100644
--- a/apps/web/src/routes/_chat.bots.$botId.tsx
+++ b/apps/web/src/routes/_chat.bots.$botId.tsx
@@ -30,6 +30,7 @@ function BotThreadRouteView() {
description,
engine,
sandbox,
+ voiceEnabled,
disabledMcpServerIds,
}) => {
if (!environmentId) return false;
@@ -42,6 +43,7 @@ function BotThreadRouteView() {
description,
engine,
sandbox,
+ voiceEnabled,
disabledMcpServerIds,
},
});
diff --git a/apps/web/src/settingsDialogStore.test.ts b/apps/web/src/settingsDialogStore.test.ts
index 222ea28b4..d74d58069 100644
--- a/apps/web/src/settingsDialogStore.test.ts
+++ b/apps/web/src/settingsDialogStore.test.ts
@@ -28,6 +28,7 @@ describe("settings dialog store", () => {
describe("legacy settings deep links", () => {
it("maps a known settings path onto its section", () => {
expect(settingsSectionFromPathname("/settings/connections")).toBe("connections");
+ expect(settingsSectionFromPathname("/settings/voice")).toBe("voice");
expect(settingsSectionFromPathname("/settings/source-control")).toBe("source-control");
});
diff --git a/apps/web/src/settingsDialogStore.ts b/apps/web/src/settingsDialogStore.ts
index 145693454..59b2cfb76 100644
--- a/apps/web/src/settingsDialogStore.ts
+++ b/apps/web/src/settingsDialogStore.ts
@@ -13,6 +13,7 @@ export const SETTINGS_SECTIONS = [
"general",
"appearance",
"providers",
+ "voice",
"connections",
"keybindings",
"source-control",
diff --git a/docs/user/bots.md b/docs/user/bots.md
index e0f7b90ea..9e3e85299 100644
--- a/docs/user/bots.md
+++ b/docs/user/bots.md
@@ -6,11 +6,20 @@ Open a bot to edit its profile in the panel beside the conversation.
- Set the bot name.
- Add an optional label and description.
- Choose the model that the bot uses.
+- Turn on **Voice calls** to show the phone button in that bot's chat header.
- Open **Tools** to choose which installed plugins and MCP servers the bot can use. New tools are enabled for every bot by default.
-- Select **Save** to apply profile, model, and tool changes.
+- Select **Save** to apply profile, model, voice, and tool changes.
Akeru Bot stores the profile in the connected environment. Other clients connected to the same environment see the same bot configuration. Workspace-disabled tools stay unavailable to all bots.
Use the panel button to collapse or reopen the editor. The default shortcut is **Mod+Alt+B**, and you can change `Right Panel: Toggle` in the keybinding settings. On a narrow screen, the same button opens a sheet.
Bot replies support headings, links, tables, task lists, code blocks, math, and Mermaid diagrams.
+
+## Call a bot
+
+Open **Settings → Voice** to enable voice, choose the ChatGPT subscription provider, and select a voice. Turning voice off there removes calling for every bot.
+
+Turn on **Voice calls** for the bot, then select the phone button in its chat header. Akeru Bot uses the microphone and speaker on this computer with the connected ChatGPT subscription. New calls use the voice selected in Settings.
+
+Only one call can run at a time. The call bar stays visible when you open another bot. Select the bar to return to the call, or select the hang-up button to end it.
diff --git a/packages/client-runtime/src/operations/commands.ts b/packages/client-runtime/src/operations/commands.ts
index 33ab2920a..2935fc41c 100644
--- a/packages/client-runtime/src/operations/commands.ts
+++ b/packages/client-runtime/src/operations/commands.ts
@@ -61,6 +61,7 @@ export type UpdateThreadMetadataInput = CommandInput<"thread.meta.update">;
export type SetThreadRuntimeModeInput = CommandInput<"thread.runtime-mode.set">;
export type SetThreadInteractionModeInput = CommandInput<"thread.interaction-mode.set">;
export type StartThreadTurnInput = CommandInput<"thread.turn.start">;
+export type AppendVoiceTranscriptInput = CommandInput<"thread.voice-transcript.append">;
export type InterruptThreadTurnInput = CommandInput<"thread.turn.interrupt">;
export type RespondToThreadApprovalInput = CommandInput<"thread.approval.respond">;
export type RespondToThreadUserInputInput = CommandInput<"thread.user-input.respond">;
@@ -462,6 +463,17 @@ export const startThreadTurn: (input: StartThreadTurnInput) => CommandEffect = E
});
});
+export const appendVoiceTranscript: (input: AppendVoiceTranscriptInput) => CommandEffect =
+ Effect.fn("EnvironmentCommands.appendVoiceTranscript")(function* (input) {
+ const metadata = yield* timestampedCommandMetadata(input);
+ return yield* dispatch({
+ ...input,
+ type: "thread.voice-transcript.append",
+ commandId: metadata.commandId,
+ createdAt: metadata.createdAt,
+ });
+ });
+
export const interruptThreadTurn: (input: InterruptThreadTurnInput) => CommandEffect = Effect.fn(
"EnvironmentCommands.interruptThreadTurn",
)(function* (input) {
diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts
index e23841edb..26f1e7a6f 100644
--- a/packages/client-runtime/src/state/server.ts
+++ b/packages/client-runtime/src/state/server.ts
@@ -733,6 +733,11 @@ export function createServerEnvironmentAtoms<R, E>(
tag: WS_METHODS.subscriptionAuthList,
staleTimeMs: 5_000,
}),
+ voiceCall: createEnvironmentRpcQueryAtomFamily(runtime, {
+ label: "environment-data:server:voice-call",
+ tag: WS_METHODS.voiceCallGet,
+ staleTimeMs: 1_000,
+ }),
configProjection,
welcome: createEnvironmentRpcSubscriptionAtomFamily(runtime, {
label: "environment-data:server:welcome",
@@ -795,6 +800,22 @@ export function createServerEnvironmentAtoms<R, E>(
label: "environment-data:server:subscription-auth:logout",
tag: WS_METHODS.subscriptionAuthLogout,
}),
+ startVoiceCall: createEnvironmentRpcCommand(runtime, {
+ label: "environment-data:server:voice-call:start",
+ tag: WS_METHODS.voiceCallStart,
+ concurrency: {
+ mode: "singleFlight",
+ key: ({ environmentId }) => environmentId,
+ },
+ }),
+ hangupVoiceCall: createEnvironmentRpcCommand(runtime, {
+ label: "environment-data:server:voice-call:hangup",
+ tag: WS_METHODS.voiceCallHangup,
+ concurrency: {
+ mode: "singleFlight",
+ key: ({ environmentId }) => environmentId,
+ },
+ }),
signalProcess: createEnvironmentRpcCommand(runtime, {
label: "environment-data:server:signal-process",
tag: WS_METHODS.serverSignalProcess,
diff --git a/packages/client-runtime/src/state/shellReducer.test.ts b/packages/client-runtime/src/state/shellReducer.test.ts
index d90ff5403..8e985ec16 100644
--- a/packages/client-runtime/src/state/shellReducer.test.ts
+++ b/packages/client-runtime/src/state/shellReducer.test.ts
@@ -47,11 +47,15 @@ const stubBot = {
id: BotId.make("bot-1"),
name: "Builder",
title: "Backend engineer",
+ label: null,
+ description: null,
+ disabledMcpServerIds: [],
avatar: { kind: "dither" as const, seed: "builder" },
engine: null,
sandbox: "local" as const,
runtimeMode: "full-access" as const,
usageCap: null,
+ voiceEnabled: false,
groupId: null,
archivedAt: null,
createdAt: "2026-04-01T00:00:00.000Z",
diff --git a/packages/client-runtime/src/state/threadCommands.ts b/packages/client-runtime/src/state/threadCommands.ts
index c54064428..27ee94053 100644
--- a/packages/client-runtime/src/state/threadCommands.ts
+++ b/packages/client-runtime/src/state/threadCommands.ts
@@ -8,6 +8,7 @@ import {
createEnvironmentRpcCommand,
} from "./runtime.ts";
import {
+ type AppendVoiceTranscriptInput,
type ArchiveThreadInput,
type CreateThreadInput,
type DeleteThreadInput,
@@ -28,6 +29,7 @@ import {
type UnsettleThreadInput,
type UnsnoozeThreadInput,
type UpdateThreadMetadataInput,
+ appendVoiceTranscript,
archiveThread,
createThread,
deleteThread,
@@ -174,6 +176,12 @@ export function createThreadEnvironmentAtoms<R, E>(
scheduler,
concurrency,
}),
+ appendVoiceTranscript: createEnvironmentCommand(runtime, {
+ label: "environment-data:commands:thread:append-voice-transcript",
+ execute: (input: AppendVoiceTranscriptInput) => appendVoiceTranscript(input),
+ scheduler,
+ concurrency,
+ }),
interruptTurn: createEnvironmentCommand(runtime, {
label: "environment-data:commands:thread:interrupt-turn",
execute: (input: InterruptThreadTurnInput) => interruptThreadTurn(input),
diff --git a/packages/contracts/src/botConfig.test.ts b/packages/contracts/src/botConfig.test.ts
index 28dfabce5..ce475037d 100644
--- a/packages/contracts/src/botConfig.test.ts
+++ b/packages/contracts/src/botConfig.test.ts
@@ -35,7 +35,7 @@ describe("BotUsageCap", () => {
expect(bot.label).toBeNull();
expect(bot.description).toBeNull();
expect(bot.disabledMcpServerIds).toEqual([]);
- expect(bot.disabledMcpServerIds).toEqual([]);
+ expect(bot.voiceEnabled).toBe(false);
});
it("accepts cloud sandbox providers on bot events", () => {
diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts
index a289b2ac4..6b4f95c8d 100644
--- a/packages/contracts/src/index.ts
+++ b/packages/contracts/src/index.ts
@@ -31,4 +31,5 @@ export * from "./preview.ts";
export * from "./previewAutomation.ts";
export * from "./resourceTelemetry.ts";
export * from "./usage.ts";
+export * from "./voiceCall.ts";
export * from "./rpc.ts";
diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts
index 4e62c1d2e..c857a7d49 100644
--- a/packages/contracts/src/orchestration.ts
+++ b/packages/contracts/src/orchestration.ts
@@ -322,6 +322,7 @@ export const OrchestrationBot = Schema.Struct({
sandbox: Schema.NullOr(BotSandbox),
runtimeMode: RuntimeMode.pipe(Schema.withDecodingDefault(Effect.succeed(DEFAULT_RUNTIME_MODE))),
usageCap: Schema.NullOr(BotUsageCap).pipe(Schema.withDecodingDefault(Effect.succeed(null))),
+ voiceEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),
groupId: Schema.NullOr(GroupId),
archivedAt: Schema.NullOr(IsoDateTime),
createdAt: IsoDateTime,
@@ -836,6 +837,7 @@ const BotCreateCommand = Schema.Struct({
sandbox: Schema.NullOr(BotSandbox),
runtimeMode: RuntimeMode.pipe(Schema.withDecodingDefault(Effect.succeed(DEFAULT_RUNTIME_MODE))),
usageCap: Schema.NullOr(BotUsageCap).pipe(Schema.withDecodingDefault(Effect.succeed(null))),
+ voiceEnabled: Schema.optional(Schema.Boolean),
groupId: Schema.NullOr(GroupId),
createdAt: IsoDateTime,
});
@@ -854,6 +856,7 @@ const BotUpdateCommand = Schema.Struct({
sandbox: Schema.optional(Schema.NullOr(BotSandbox)),
runtimeMode: Schema.optional(RuntimeMode),
usageCap: Schema.optional(Schema.NullOr(BotUsageCap)),
+ voiceEnabled: Schema.optional(Schema.Boolean),
groupId: Schema.optional(Schema.NullOr(GroupId)),
});
@@ -1194,6 +1197,17 @@ const ClientThreadTurnStartCommand = Schema.Struct({
createdAt: IsoDateTime,
});
+export const ThreadVoiceTranscriptAppendCommand = Schema.Struct({
+ type: Schema.Literal("thread.voice-transcript.append"),
+ commandId: CommandId,
+ threadId: ThreadId,
+ messageId: MessageId,
+ role: Schema.Literals(["user", "assistant"]),
+ text: TrimmedNonEmptyString,
+ respondingBotId: Schema.optional(BotId),
+ createdAt: IsoDateTime,
+});
+
const ThreadTurnInterruptCommand = Schema.Struct({
type: Schema.Literal("thread.turn.interrupt"),
commandId: CommandId,
@@ -1275,6 +1289,7 @@ const DispatchableClientOrchestrationCommand = Schema.Union([
ThreadRuntimeModeSetCommand,
ThreadInteractionModeSetCommand,
ThreadTurnStartCommand,
+ ThreadVoiceTranscriptAppendCommand,
ThreadTurnInterruptCommand,
ThreadApprovalRespondCommand,
ThreadUserInputRespondCommand,
@@ -1318,6 +1333,7 @@ export const ClientOrchestrationCommand = Schema.Union([
ThreadRuntimeModeSetCommand,
ThreadInteractionModeSetCommand,
ClientThreadTurnStartCommand,
+ ThreadVoiceTranscriptAppendCommand,
ThreadTurnInterruptCommand,
ThreadApprovalRespondCommand,
ThreadUserInputRespondCommand,
@@ -1521,6 +1537,7 @@ export const BotCreatedPayload = Schema.Struct({
sandbox: Schema.NullOr(BotSandbox),
runtimeMode: RuntimeMode.pipe(Schema.withDecodingDefault(Effect.succeed(DEFAULT_RUNTIME_MODE))),
usageCap: Schema.NullOr(BotUsageCap).pipe(Schema.withDecodingDefault(Effect.succeed(null))),
+ voiceEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),
groupId: Schema.NullOr(GroupId),
createdAt: IsoDateTime,
updatedAt: IsoDateTime,
@@ -1538,6 +1555,7 @@ export const BotUpdatedPayload = Schema.Struct({
sandbox: Schema.optional(Schema.NullOr(BotSandbox)),
runtimeMode: Schema.optional(RuntimeMode),
usageCap: Schema.optional(Schema.NullOr(BotUsageCap)),
+ voiceEnabled: Schema.optional(Schema.Boolean),
groupId: Schema.optional(Schema.NullOr(GroupId)),
updatedAt: IsoDateTime,
});
diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts
index 33ec8ede5..46508f5b2 100644
--- a/packages/contracts/src/rpc.ts
+++ b/packages/contracts/src/rpc.ts
@@ -189,6 +189,13 @@ import {
SourceControlRepositoryLookupInput,
} from "./sourceControl.ts";
import { VcsError } from "./vcs.ts";
+import {
+ VoiceCallError,
+ VoiceCallHangupInput,
+ VoiceCallSnapshot,
+ VoiceCallStartInput,
+ VoiceCallStartResult,
+} from "./voiceCall.ts";
export const WS_METHODS = {
// Project registry methods
@@ -270,6 +277,9 @@ export const WS_METHODS = {
subscriptionAuthComplete: "subscriptionAuth.complete",
subscriptionAuthCancel: "subscriptionAuth.cancel",
subscriptionAuthLogout: "subscriptionAuth.logout",
+ voiceCallGet: "voiceCall.get",
+ voiceCallStart: "voiceCall.start",
+ voiceCallHangup: "voiceCall.hangup",
serverDiscoverSourceControl: "server.discoverSourceControl",
serverGetTraceDiagnostics: "server.getTraceDiagnostics",
serverGetProcessDiagnostics: "server.getProcessDiagnostics",
@@ -412,6 +422,24 @@ export const WsSubscriptionAuthLogoutRpc = Rpc.make(WS_METHODS.subscriptionAuthL
error: Schema.Union([SubscriptionAuthError, EnvironmentAuthorizationError]),
});
+export const WsVoiceCallGetRpc = Rpc.make(WS_METHODS.voiceCallGet, {
+ payload: Schema.Struct({}),
+ success: VoiceCallSnapshot,
+ error: EnvironmentAuthorizationError,
+});
+
+export const WsVoiceCallStartRpc = Rpc.make(WS_METHODS.voiceCallStart, {
+ payload: VoiceCallStartInput,
+ success: VoiceCallStartResult,
+ error: Schema.Union([VoiceCallError, EnvironmentAuthorizationError]),
+});
+
+export const WsVoiceCallHangupRpc = Rpc.make(WS_METHODS.voiceCallHangup, {
+ payload: VoiceCallHangupInput,
+ success: VoiceCallSnapshot,
+ error: Schema.Union([VoiceCallError, EnvironmentAuthorizationError]),
+});
+
export const WsServerDiscoverSourceControlRpc = Rpc.make(WS_METHODS.serverDiscoverSourceControl, {
payload: Schema.Struct({}),
success: SourceControlDiscoveryResult,
@@ -917,6 +945,9 @@ export const WsRpcGroup = RpcGroup.make(
WsSubscriptionAuthCompleteRpc,
WsSubscriptionAuthCancelRpc,
WsSubscriptionAuthLogoutRpc,
+ WsVoiceCallGetRpc,
+ WsVoiceCallStartRpc,
+ WsVoiceCallHangupRpc,
WsServerDiscoverSourceControlRpc,
WsServerGetTraceDiagnosticsRpc,
WsServerGetProcessDiagnosticsRpc,
diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts
index d75b0b4ae..d555ce747 100644
--- a/packages/contracts/src/settings.test.ts
+++ b/packages/contracts/src/settings.test.ts
@@ -20,6 +20,28 @@ const decodeServerSettingsPatch = Schema.decodeUnknownSync(ServerSettingsPatch);
const encodeServerSettings = Schema.encodeSync(ServerSettings);
const decodeClaudeSettings = Schema.decodeUnknownSync(ClaudeSettings);
+describe("ServerSettings voice", () => {
+ it("defaults to enabled ChatGPT voice calls with Alloy", () => {
+ expect(decodeServerSettings({}).voice).toEqual({
+ enabled: true,
+ provider: "chatgpt",
+ voice: "alloy",
+ });
+ });
+
+ it("accepts a partial voice settings patch", () => {
+ expect(decodeServerSettingsPatch({ voice: { enabled: false, voice: "marin" } }).voice).toEqual({
+ enabled: false,
+ voice: "marin",
+ });
+ });
+
+ it("rejects unsupported providers and voices", () => {
+ expect(() => decodeServerSettings({ voice: { provider: "xai" } })).toThrow();
+ expect(() => decodeServerSettingsPatch({ voice: { voice: "unknown" } })).toThrow();
+ });
+});
+
describe("ClaudeSettings auto-compaction", () => {
it("uses Claude's default threshold when no override is configured", () => {
expect(decodeClaudeSettings({}).autoCompactWindow).toBe("");
diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts
index fec022d5d..4bf61e8bc 100644
--- a/packages/contracts/src/settings.ts
+++ b/packages/contracts/src/settings.ts
@@ -18,6 +18,7 @@ import {
PreviewViewportSetting,
PreviewZoomFactor,
} from "./preview.ts";
+import { VoiceProvider, ChatGptRealtimeVoice, VoiceSettings } from "./voiceCall.ts";
import {
ProviderInstanceConfig,
ProviderInstanceId,
@@ -650,6 +651,7 @@ export const ServerSettings = Schema.Struct({
* between a desktop window and a phone attached to the same server.
*/
enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
+ voice: VoiceSettings,
backgroundActivity: BackgroundActivitySettings,
// Legacy flat fields retained for old settings files and old clients. New
// consumers should resolve `backgroundActivity` instead.
@@ -864,6 +866,13 @@ export const ServerSettingsPatch = Schema.Struct({
enableLegacyTokenStreaming: Schema.optionalKey(Schema.Boolean),
enableProviderUpdateChecks: Schema.optionalKey(Schema.Boolean),
enableAgentBrowserAccess: Schema.optionalKey(Schema.Boolean),
+ voice: Schema.optionalKey(
+ Schema.Struct({
+ enabled: Schema.optionalKey(Schema.Boolean),
+ provider: Schema.optionalKey(VoiceProvider),
+ voice: Schema.optionalKey(ChatGptRealtimeVoice),
+ }),
+ ),
backgroundActivity: Schema.optionalKey(
Schema.Struct({
schemaVersion: Schema.optionalKey(Schema.Literal(1)),
diff --git a/packages/contracts/src/voiceCall.test.ts b/packages/contracts/src/voiceCall.test.ts
new file mode 100644
index 000000000..f2cd3977a
--- /dev/null
+++ b/packages/contracts/src/voiceCall.test.ts
@@ -0,0 +1,29 @@
+import * as Schema from "effect/Schema";
+import { describe, expect, it } from "vite-plus/test";
+
+import { BotId } from "./baseSchemas.ts";
+import { VoiceCallStartInput, VoiceCallStartResult } from "./voiceCall.ts";
+
+const sdp = "v=0\r\nm=audio 9 UDP/TLS/RTP/SAVPF 111\r\n";
+
+describe("voice call contracts", () => {
+ it("preserves SDP line endings across the RPC boundary", () => {
+ const input = Schema.decodeUnknownSync(VoiceCallStartInput)({
+ botId: BotId.make("bot-voice"),
+ sdp,
+ });
+ expect(input.sdp).toBe(sdp);
+
+ const result = Schema.decodeUnknownSync(VoiceCallStartResult)({
+ call: {
+ callId: "call-1",
+ status: "live",
+ botId: BotId.make("bot-voice"),
+ botName: "Akeru",
+ startedAt: "2026-08-27T00:00:00.000Z",
+ },
+ answerSdp: sdp,
+ });
+ expect(result.answerSdp).toBe(sdp);
+ });
+});
diff --git a/packages/contracts/src/voiceCall.ts b/packages/contracts/src/voiceCall.ts
new file mode 100644
index 000000000..c4f273f16
--- /dev/null
+++ b/packages/contracts/src/voiceCall.ts
@@ -0,0 +1,72 @@
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+
+import { BotId, IsoDateTime, TrimmedNonEmptyString } from "./baseSchemas.ts";
+
+export const CHATGPT_REALTIME_VOICE_MODEL = "gpt-realtime-2";
+export const CHATGPT_REALTIME_VOICES = [
+ "alloy",
+ "ash",
+ "ballad",
+ "coral",
+ "echo",
+ "sage",
+ "shimmer",
+ "verse",
+ "marin",
+ "cedar",
+] as const;
+export const ChatGptRealtimeVoice = Schema.Literals(CHATGPT_REALTIME_VOICES);
+export type ChatGptRealtimeVoice = typeof ChatGptRealtimeVoice.Type;
+
+export const VoiceProvider = Schema.Literal("chatgpt");
+export type VoiceProvider = typeof VoiceProvider.Type;
+
+export const VoiceSettings = Schema.Struct({
+ enabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
+ provider: VoiceProvider.pipe(Schema.withDecodingDefault(Effect.succeed("chatgpt" as const))),
+ voice: ChatGptRealtimeVoice.pipe(Schema.withDecodingDefault(Effect.succeed("alloy" as const))),
+}).pipe(Schema.withDecodingDefault(Effect.succeed({})));
+export type VoiceSettings = typeof VoiceSettings.Type;
+
+export const VoiceCallIdle = Schema.Struct({ status: Schema.Literal("idle") });
+
+export const VoiceCallActive = Schema.Struct({
+ callId: TrimmedNonEmptyString,
+ status: Schema.Literals(["starting", "live"]),
+ botId: BotId,
+ botName: TrimmedNonEmptyString,
+ startedAt: IsoDateTime,
+});
+
+export const VoiceCallSnapshot = Schema.Union([VoiceCallIdle, VoiceCallActive]);
+export type VoiceCallSnapshot = typeof VoiceCallSnapshot.Type;
+
+export const VoiceSessionDescription = Schema.String.check(Schema.isNonEmpty());
+
+export const VoiceCallStartInput = Schema.Struct({
+ botId: BotId,
+ sdp: VoiceSessionDescription,
+});
+export type VoiceCallStartInput = typeof VoiceCallStartInput.Type;
+
+export const VoiceCallHangupInput = Schema.Struct({ callId: TrimmedNonEmptyString });
+export type VoiceCallHangupInput = typeof VoiceCallHangupInput.Type;
+
+export const VoiceCallStartResult = Schema.Struct({
+ call: VoiceCallActive,
+ answerSdp: VoiceSessionDescription,
+});
+export type VoiceCallStartResult = typeof VoiceCallStartResult.Type;
+
+export class VoiceCallError extends Schema.TaggedErrorClass<VoiceCallError>()("VoiceCallError", {
+ reason: Schema.Literals([
+ "already-active",
+ "bot-not-found",
+ "voice-disabled",
+ "subscription-unavailable",
+ "upstream-failed",
+ "call-not-active",
+ ]),
+ message: TrimmedNonEmptyString,
+}) {}
diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts
index d06c892c4..a1c0d90d3 100644
--- a/scripts/build-desktop-artifact.test.ts
+++ b/scripts/build-desktop-artifact.test.ts
@@ -1098,6 +1098,7 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => {
assert.include(entitlements, "<string>webcredentials:clerk.example.com</string>");
assert.include(entitlements, "<string>webcredentials:example.clerk.accounts.dev</string>");
assert.include(entitlements, "<key>com.apple.security.cs.allow-jit</key>");
+ assert.include(entitlements, "<key>com.apple.security.device.audio-input</key>");
});
it("rejects incomplete macOS passkey signing configuration", () => {
diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts
index 7377fd0f0..3142b4d0a 100644
--- a/scripts/build-desktop-artifact.ts
+++ b/scripts/build-desktop-artifact.ts
@@ -1071,6 +1071,8 @@ ${associatedDomains}
<true/>
<key>com.apple.security.cs.disable-library-validation</key>
<true/>
+ <key>com.apple.security.device.audio-input</key>
+ <true/>
</dict>
</plist>
`;
@@ -2118,6 +2120,9 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* (
target: target === "dmg" ? [target, "zip"] : [target],
icon: "icon.icns",
category: "public.app-category.developer-tools",
+ extendInfo: {
+ NSMicrophoneUsageDescription: "Akeru Bot uses the microphone for calls with your bots.",
+ },
protocols: [
{
name: "Akeru Bot",