Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
import * as Match from "effect/Match";
import * as Predicate from "effect/Predicate";
import * as NodeCrypto from "node:crypto";
import {
VoiceCallError,
type BotId,
type VoiceCallSnapshot,
type VoiceCallStartInput,
type VoiceCallStartResult,
type VoiceSettings,
type VoiceApiProvider,
type VoiceTranscribeInput,
type VoiceSynthesizeInput,
type VoiceListVoicesResult,
VOICE_API_PROVIDERS,
VoiceTranscribeInput as TranscribeSchema,
VoiceSynthesizeInput as SynthesizeSchema,
type OrchestrationEvent,
} from "@akeru/contracts";
import * as Context from "effect/Context";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Semaphore from "effect/Semaphore";
import * as Stream from "effect/Stream";
import * as Schema from "effect/Schema";
import * as ServerConfig from "../config.ts";
import { ServerSecretStore } from "../auth/ServerSecretStore.ts";
import {
classifyVoiceFailure,
voiceAdapters,
voiceFailure,
type VoiceAdapters,
} from "./VoiceAdapters.ts";
import { ProjectionBotRepository } from "../persistence/Services/ProjectionBots.ts";
import { ServerSettingsService } from "../serverSettings.ts";
import { SubscriptionAuthService } from "../subscription-auth/service.ts";
import { type ActiveVoiceCall, snapshot } from "./VoiceCallState.ts";
import {
type ChatGptRealtimeSession,
getCodexCliCredential,
defaultSession,
instructionsForBot,
} from "./ChatGptRealtime.ts";
const isVoiceCallError = Schema.is(VoiceCallError);
const isTranscribeInput = Schema.is(TranscribeSchema);
const isSynthesizeInput = Schema.is(SynthesizeSchema);
export interface VoiceCallManagerOptions {
readonly adapters?: VoiceAdapters;
readonly makeSession?: () => ChatGptRealtimeSession;
readonly getCredential?: () => Promise<
{ readonly accessToken: string; readonly accountId: string } | undefined
>;
readonly getCodexCliCredential?: () => Promise<
{ readonly accessToken: string; readonly accountId: string } | undefined
>;
}
export class VoiceCallManager extends Context.Service<
VoiceCallManager,
{
readonly get: Effect.Effect<VoiceCallSnapshot>;
readonly providers: Effect.Effect<
{
providers: ReadonlyArray<{
provider: VoiceApiProvider;
connected: boolean;
keyRejected: boolean;
}>;
},
VoiceCallError
>;
readonly connect: (
provider: VoiceApiProvider,
apiKey: string,
) => Effect.Effect<{ provider: VoiceApiProvider; connected: boolean }, VoiceCallError>;
readonly disconnect: (
provider: VoiceApiProvider,
) => Effect.Effect<{ provider: VoiceApiProvider; connected: boolean }, VoiceCallError>;
readonly test: (
provider: VoiceApiProvider,
) => Effect.Effect<{ provider: VoiceApiProvider; connected: boolean }, VoiceCallError>;
readonly listVoices: (
provider: VoiceApiProvider,
cursor?: string,
) => Effect.Effect<VoiceListVoicesResult, VoiceCallError>;
readonly transcribe: (
input: VoiceTranscribeInput,
ownerId: string,
) => Effect.Effect<{ text: string }, VoiceCallError>;
readonly synthesize: (
input: VoiceSynthesizeInput,
ownerId: string,
) => Effect.Effect<{ audioBase64: string; mimeType: "audio/mpeg" }, VoiceCallError>;
readonly cancel: (
operationId: string,
ownerId: string,
) => Effect.Effect<{ cancelled: boolean }>;
readonly start: (
input: VoiceCallStartInput,
ownerId: string,
) => Effect.Effect<VoiceCallStartResult, VoiceCallError>;
readonly hangup: (
callId: string,
ownerId: string,
) => Effect.Effect<VoiceCallSnapshot, VoiceCallError>;
readonly hangupOwner: (ownerId: string) => Effect.Effect<void>;
/** Ends the active call when it belongs to `botId`, freeing the single call slot. */
readonly hangupBot: (botId: BotId) => Effect.Effect<void>;
}
>()("akeru-bot/voiceCall/VoiceCallManager") {}
const make = (options?: VoiceCallManagerOptions) =>
Effect.gen(function* () {
const config = yield* ServerConfig.ServerConfig;
const bots = yield* ProjectionBotRepository;
const serverSettings = yield* ServerSettingsService;
const lock = yield* Semaphore.make(1);
const auth = yield* SubscriptionAuthService.forSecretsDir(config.secretsDir);
let active: ActiveVoiceCall | null = null;
const secrets = yield* Effect.serviceOption(ServerSecretStore);
const adapters = options?.adapters ?? voiceAdapters();
const operations = new Map<
string,
{ ownerId: string; controller: AbortController; provider?: VoiceApiProvider }
>();
const operationKey = (ownerId: string, id: string) => JSON.stringify([ownerId, id]);
const secretName = (provider: VoiceApiProvider) => `voice-${provider}`;
const rejectedSecretName = (provider: VoiceApiProvider) => `voice-${provider}-rejected`;
// Bumped whenever a provider's key is replaced or removed, so a Test that read
// an earlier connection cannot record its verdict against a later one, even
// when that later connection restored the same key.
const generations = new Map<VoiceApiProvider, number>();
const bumpGeneration = (provider: VoiceApiProvider) =>
generations.set(provider, (generations.get(provider) ?? 0) + 1);
const getKey = Effect.fn("VoiceCallManager.getKey")(function* (provider: VoiceApiProvider) {
if (Option.isNone(secrets)) return yield* voiceFailure("provider-unavailable");
const value = yield* secrets.value
.get(secretName(provider))
.pipe(Effect.mapError(() => voiceFailure("provider-unavailable")));
if (Option.isNone(value)) return yield* voiceFailure("provider-unavailable");
return new TextDecoder().decode(value.value);
});
const selectedProviders = (settings: VoiceSettings): ReadonlyArray<VoiceApiProvider> =>
Match.value(settings).pipe(
Match.when({ provider: "chatgpt" }, (_settings) => []),
Match.when({ provider: "openai" }, (): ReadonlyArray<VoiceApiProvider> => ["openai"]),
Match.orElse((settings) => [
...new Set([
settings.transcriptionProvider ?? "openai",
settings.synthesisProvider ?? "openai",
]),
]),
);
const selectedVoice = (settings: VoiceSettings) => {
const provider = settings.synthesisProvider ?? "openai";
return settings.synthesisVoices?.[provider] ?? (provider === "openai" ? "alloy" : undefined);
};
const assertMutable = Effect.fn("VoiceCallManager.assertMutable")(function* (
provider: VoiceApiProvider,
) {
if (active && selectedProviders(active.settings).includes(provider))
return yield* voiceFailure("provider-in-use");
if (Option.isNone(secrets)) return yield* voiceFailure("provider-unavailable");
return secrets.value;
});
const connect = (provider: VoiceApiProvider, apiKey: string) =>
lock.withPermits(1)(
Effect.gen(function* () {
const store = yield* assertMutable(provider);
if (!apiKey.trim() || apiKey.length > 4096 || /[\r\n]/.test(apiKey))
return yield* voiceFailure("invalid-input");
const key = new TextEncoder().encode(apiKey.trim());
yield* Effect.gen(function* () {
const saved = yield* store.get(secretName(provider));
yield* store.set(secretName(provider), key);
// A replaced key drops the old rejection, so restoring that key later
// waits for a new Test instead of reviving the stale verdict.
if (Option.isNone(saved) || keyDigest(saved.value) !== keyDigest(key)) {
bumpGeneration(provider);
yield* store.remove(rejectedSecretName(provider));
}
}).pipe(Effect.mapError(() => voiceFailure("provider-unavailable")));
return { provider, connected: true };
}),
);
const disconnect = (provider: VoiceApiProvider) =>
lock.withPermits(1)(
Effect.gen(function* () {
const store = yield* assertMutable(provider);
bumpGeneration(provider);
for (const name of [secretName(provider), rejectedSecretName(provider)]) {
yield* store
.remove(name)
.pipe(Effect.mapError(() => voiceFailure("provider-unavailable")));
}
// Work that already read the removed key must not finish after the disconnect.
for (const operation of operations.values()) {
if (operation.provider === provider) operation.controller.abort();
}
return { provider, connected: false };
}),
);
// Digest of the saved key each provider rejected on its last Test. It lives
// beside the key so every client, and a restarted server, sees the same
// verdict, and a replaced key starts without one.
const keyDigest = (key: Uint8Array | string) =>
NodeCrypto.createHash("sha256").update(key).digest("hex");
const providers = Effect.gen(function* () {
const result = [];
for (const provider of VOICE_API_PROVIDERS) {
const read = (name: string) =>
Option.isSome(secrets)
? secrets.value
.get(name)
.pipe(Effect.mapError(() => voiceFailure("provider-unavailable")))
: Effect.succeed(Option.none<Uint8Array>());
const key = yield* read(secretName(provider));
const rejected = Option.isSome(key)
? yield* read(rejectedSecretName(provider))
: Option.none<Uint8Array>();
result.push({
provider,
connected: Option.isSome(key),
keyRejected:
Option.isSome(key) &&
Option.isSome(rejected) &&
new TextDecoder().decode(rejected.value) === keyDigest(key.value),
});
}
return { providers: result };
});
// Records a Test verdict only while the tested connection is still the saved
// one, so a slow Test of a replaced key cannot overwrite the replacement's verdict.
const recordVerdict = (
provider: VoiceApiProvider,
tested: { key: string; generation: number },
rejected: boolean,
) =>
lock.withPermits(1)(
Effect.gen(function* () {
if (Option.isNone(secrets)) return;
if ((generations.get(provider) ?? 0) !== tested.generation) return;
const { key } = tested;
const store = secrets.value;
const current = yield* store.get(secretName(provider));
if (Option.isNone(current) || keyDigest(current.value) !== keyDigest(key)) return;
yield* rejected
? store.set(rejectedSecretName(provider), new TextEncoder().encode(keyDigest(key)))
: store.remove(rejectedSecretName(provider));
}).pipe(Effect.mapError(() => voiceFailure("provider-unavailable"))),
);
const test = Effect.fn("VoiceCallManager.test")(function* (provider: VoiceApiProvider) {
const tested = yield* lock.withPermits(1)(
Effect.map(getKey(provider), (key) => ({
key,
generation: generations.get(provider) ?? 0,
})),
);
yield* Effect.tryPromise({
try: (signal) => adapters.test(provider, tested.key, signal),
catch: (cause) => classifyVoiceFailure(cause),
}).pipe(
Effect.tapError((error) =>
error.reason === "provider-auth"
? recordVerdict(provider, tested, true).pipe(Effect.ignore)
: Effect.void,
),
);
yield* recordVerdict(provider, tested, false);
return { provider, connected: true, keyRejected: false };
});
const listVoices = Effect.fn("VoiceCallManager.listVoices")(function* (
provider: VoiceApiProvider,
cursor?: string,
) {
const key = yield* getKey(provider);
return yield* Effect.tryPromise({
try: (signal) => adapters.listVoices(provider, key, signal, cursor),
catch: (cause) => classifyVoiceFailure(cause),
});
});
const cancel = (id: string, ownerId: string) =>
Effect.sync(() => {
const operation = operations.get(operationKey(ownerId, id));
operation?.controller.abort();
return { cancelled: operation !== undefined };
});
const audioOperation = <A>(
input: { operationId: string; callId?: string },
ownerId: string,
capability: "transcription" | "synthesis",
run: (settings: VoiceSettings, key: string, signal: AbortSignal) => Promise<A>,
) =>
Effect.gen(function* () {
const id = operationKey(ownerId, input.operationId);
const controller = new AbortController();
yield* Effect.addFinalizer(() =>
Effect.sync(() => {
controller.abort();
if (operations.get(id)?.controller === controller) operations.delete(id);
}),
);
const pinned = yield* lock.withPermits(1)(
Effect.gen(function* () {
if (operations.size >= 8 || operations.has(id)) return yield* voiceFailure("busy");
if (input.callId === undefined && active && active.ownerId !== ownerId)
return yield* voiceFailure("call-not-active");
if (
input.callId !== undefined &&
(active?.callId !== input.callId ||
active.ownerId !== ownerId ||
active.status !== "live" ||
active.settings.provider !== "composed")
)
return yield* voiceFailure("call-not-active");
operations.set(id, { ownerId, controller });
const call = input.callId !== undefined ? active : null;
const settings =
call?.settings ??
(yield* serverSettings.getSettings.pipe(
Effect.map((s) => s.voice),
Effect.mapError(() => voiceFailure()),
));
if (!settings.enabled) return yield* voiceFailure("voice-disabled");
const provider =
capability === "transcription"
? (settings.transcriptionProvider ?? "openai")
: (settings.synthesisProvider ?? "openai");
const key = call ? call.credentials[provider] : yield* getKey(provider);
if (!key) return yield* voiceFailure("provider-unavailable");
if (!call) operations.set(id, { ownerId, controller, provider });
if (controller.signal.aborted) return yield* voiceFailure("cancelled");
return { settings, key, callSignal: call?.abortController.signal };
}),
);
return yield* Effect.tryPromise({
try: (signal) =>
run(
pinned.settings,
pinned.key,
AbortSignal.any([
signal,
controller.signal,
AbortSignal.timeout(60_000),
...(pinned.callSignal ? [pinned.callSignal] : []),
]),
),
catch: (cause) => (isVoiceCallError(cause) ? voiceFailure(cause.reason) : voiceFailure()),
});
}).pipe(Effect.scoped);
const transcribe = (input: VoiceTranscribeInput, ownerId: string) =>
Effect.gen(function* () {
if (!isTranscribeInput(input)) return yield* voiceFailure("invalid-input");
return yield* audioOperation(input, ownerId, "transcription", (settings, key, signal) =>
adapters.transcribe(settings.transcriptionProvider ?? "openai", key, input, signal),
);
});
const synthesize = (input: VoiceSynthesizeInput, ownerId: string) =>
Effect.gen(function* () {
if (!isSynthesizeInput(input)) return yield* voiceFailure("invalid-input");
return yield* audioOperation(input, ownerId, "synthesis", async (settings, key, signal) => {
const provider = settings.synthesisProvider ?? "openai";
const voice = selectedVoice(settings);
if (!voice) throw voiceFailure("invalid-voice");
if (!input.callId) await adapters.validateVoice(provider, key, voice, signal);
return adapters.synthesize(provider, key, voice, input.text, signal);
});
});
const clear = (callId: string) =>
lock.withPermits(1)(
Effect.sync(() => {
if (active?.callId !== callId) return;
active.abortController.abort();
active = null;
}),
);
const start = Effect.fn("VoiceCallManager.start")(function* (
input: VoiceCallStartInput,
ownerId: string,
) {
const voiceSettings = yield* serverSettings.getSettings.pipe(
Effect.map((settings) => settings.voice),
Effect.mapError(
() =>
new VoiceCallError({
reason: "voice-disabled",
message: "Voice settings are unavailable.",
}),
),
);
if (!voiceSettings.enabled) {
return yield* new VoiceCallError({
reason: "voice-disabled",
message: "Voice calls are disabled in Settings.",
});
}
if (voiceSettings.provider !== "composed" && (!input.sdp || input.sdp.length > 65_536))
return yield* voiceFailure("invalid-input");
const claimed = yield* lock.withPermits(1)(
Effect.gen(function* () {
if (active !== null) {
return yield* new VoiceCallError({
reason: "already-active",
message: `A call with ${active.botName} is already active. Hang up before starting another call.`,
});
}
const bot = yield* bots.getById({ botId: input.botId }).pipe(
Effect.mapError(
() =>
new VoiceCallError({
reason: "bot-not-found",
message: "Could not load this bot.",
}),
),
);
if (Option.isNone(bot) || bot.value.archivedAt !== null) {
return yield* new VoiceCallError({
reason: "bot-not-found",
message: "This bot is not available.",
});
}
if (!bot.value.voiceEnabled) {
return yield* new VoiceCallError({
reason: "voice-disabled",
message: `Voice calls are disabled for ${bot.value.name}.`,
});
}
const credentials: Partial<Record<VoiceApiProvider, string>> = {};
for (const provider of selectedProviders(voiceSettings))
credentials[provider] = yield* getKey(provider);
const startedAt = DateTime.formatIso(yield* DateTime.now);
const call: ActiveVoiceCall = {
callId: NodeCrypto.randomUUID(),
ownerId,
botId: bot.value.botId,
botName: bot.value.name,
startedAt,
abortController: new AbortController(),
settings: structuredClone(voiceSettings),
credentials,
status: "starting",
};
active = call;
return { call, bot: bot.value };
}),
);
return yield* Effect.gen(function* () {
if (voiceSettings.provider !== "chatgpt") {
return yield* Effect.gen(function* () {
const answerSdp = yield* Effect.tryPromise({
try: async (signal) => {
const combined = AbortSignal.any([
signal,
claimed.call.abortController.signal,
AbortSignal.timeout(60_000),
]);
if (voiceSettings.provider === "openai") {
const key = claimed.call.credentials.openai;
if (!key || !input.sdp) throw voiceFailure("invalid-input");
return adapters.negotiate(
key,
input.sdp,
instructionsForBot(claimed.bot),
voiceSettings.openaiVoice ?? "alloy",
combined,
);
}
const provider = voiceSettings.synthesisProvider ?? "openai";
const key = claimed.call.credentials[provider];
const voice = selectedVoice(voiceSettings);
if (!key || !voice) throw voiceFailure("invalid-voice");
await adapters.validateVoice(provider, key, voice, combined);
return undefined;
},
catch: (cause) =>
isVoiceCallError(cause) ? voiceFailure(cause.reason) : voiceFailure(),
});
yield* lock.withPermits(1)(
Effect.gen(function* () {
if (active?.callId !== claimed.call.callId)
return yield* voiceFailure("call-not-active");
active.status = "live";
}),
);
return {
call: {
callId: claimed.call.callId,
status: "live" as const,
botId: claimed.call.botId,
botName: claimed.call.botName,
startedAt: claimed.call.startedAt,
},
...(answerSdp === undefined ? {} : { answerSdp }),
transport:
voiceSettings.provider === "composed" ? ("composed" as const) : ("webrtc" as const),
settings: structuredClone(claimed.call.settings),
};
}).pipe(
Effect.onExit((exit) =>
Predicate.isTagged(exit, "Success") ? Effect.void : clear(claimed.call.callId),
),
);
}
const credential = yield* Effect.tryPromise({
try: () =>
options?.getCredential
? options.getCredential()
: auth
.getOpenAICodexAccess()
.then(
(credential) =>
credential ?? (options?.getCodexCliCredential ?? getCodexCliCredential)(),
),
catch: () =>
new VoiceCallError({
reason: "subscription-unavailable",
message: "Connect a ChatGPT subscription before starting a call.",
}),
}).pipe(Effect.tapError(() => clear(claimed.call.callId)));
if (credential === undefined) {
yield* clear(claimed.call.callId);
return yield* new VoiceCallError({
reason: "subscription-unavailable",
message: "Connect a ChatGPT subscription before starting a call.",
});
}
const session = yield* Effect.try({
try: () => (options?.makeSession ?? defaultSession)(),
catch: () => voiceFailure(),
}).pipe(Effect.tapError(() => clear(claimed.call.callId)));
const answerSdp = yield* Effect.tryPromise({
try: (signal) =>
session.negotiate({
offerSdp: input.sdp ?? "",
instructions: instructionsForBot(claimed.bot),
signal: AbortSignal.any([signal, claimed.call.abortController.signal]),
voice: voiceSettings.voice,
...credential,
}),
catch: () =>
active?.callId === claimed.call.callId
? voiceFailure()
: new VoiceCallError({
reason: "call-not-active",
message: "This call is no longer active.",
}),
}).pipe(Effect.tapError(() => clear(claimed.call.callId)));
yield* lock.withPermits(1)(
Effect.gen(function* () {
if (active?.callId !== claimed.call.callId) {
return yield* new VoiceCallError({
reason: "call-not-active",
message: "This call is no longer active.",
});
}
active.status = "live";
}),
);
return {
call: {
callId: claimed.call.callId,
status: "live" as const,
botId: claimed.call.botId,
botName: claimed.call.botName,
startedAt: claimed.call.startedAt,
},
answerSdp,
transport: "webrtc" as const,
settings: structuredClone(claimed.call.settings),
};
}).pipe(
Effect.onExit((exit) =>
Predicate.isTagged(exit, "Success") ? Effect.void : clear(claimed.call.callId),
),
);
});
const hangup = (callId: string, ownerId: string) =>
lock.withPermits(1)(
Effect.gen(function* () {
if (active?.callId !== callId || active.ownerId !== ownerId) {
return yield* new VoiceCallError({
reason: "call-not-active",
message: "This call is no longer active.",
});
}
active.abortController.abort();
active = null;
return snapshot(active);
}),
);
const hangupOwner = (ownerId: string) =>
lock.withPermits(1)(
Effect.sync(() => {
for (const operation of operations.values())
if (operation.ownerId === ownerId) operation.controller.abort();
if (active?.ownerId !== ownerId) return;
active.abortController.abort();
active = null;
}),
);
const hangupBot = (botId: BotId) =>
lock.withPermits(1)(
Effect.sync(() => {
if (active?.botId !== botId) return;
active.abortController.abort();
active = null;
}),
);
yield* Effect.addFinalizer(() =>
Effect.sync(() => {
for (const operation of operations.values()) operation.controller.abort();
operations.clear();
active?.abortController.abort();
active = null;
}),
);
return VoiceCallManager.of({
get: Effect.sync(() => snapshot(active)),
providers,
connect,
disconnect,
test,
listVoices,
transcribe,
synthesize,
cancel,
start,
hangup,
hangupOwner,
hangupBot,
});
});
/** Ends a deleted bot's call as its `bot.deleted` event arrives. */
export const hangupDeletedBotCalls = <E, R>(
voiceCalls: VoiceCallManager["Service"],
events: Stream.Stream<OrchestrationEvent, E, R>,
) =>
Stream.runForEach(events, (event) =>
event.type === "bot.deleted" ? voiceCalls.hangupBot(event.payload.botId) : Effect.void,
);
export const layer = (options?: VoiceCallManagerOptions) =>
Layer.effect(VoiceCallManager, make(options));
export { parseCodexCliAuth } from "./ChatGptRealtime.ts";
export { defaultSession } from "./ChatGptRealtime.ts";