Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
import * as Effect from "effect/Effect";
import * as Schema from "effect/Schema";
import { BotId, IsoDateTime, TrimmedNonEmptyString } from "./baseSchemas.ts";
export const CHATGPT_REALTIME_VOICE_MODEL = "gpt-realtime-2.1";
export const CHATGPT_REALTIME_VOICES = [
"alloy",
"ash",
"ballad",
"coral",
"echo",
"sage",
"shimmer",
"verse",
"marin",
"cedar",
] as const;
export const ChatGptRealtimeVoice = Schema.Literals(CHATGPT_REALTIME_VOICES);
export type ChatGptRealtimeVoice = typeof ChatGptRealtimeVoice.Type;
export const VOICE_API_PROVIDERS = ["openai", "elevenlabs", "cartesia", "fish"] as const;
export const VoiceApiProvider = Schema.Literals(VOICE_API_PROVIDERS);
export type VoiceApiProvider = typeof VoiceApiProvider.Type;
export const VoiceTranscriptionProvider = Schema.Literals(["openai", "elevenlabs", "cartesia"]);
export type VoiceTranscriptionProvider = typeof VoiceTranscriptionProvider.Type;
export const VoiceId = TrimmedNonEmptyString.check(Schema.isMaxLength(256));
export const VoiceSynthesisVoices = Schema.Struct({
openai: Schema.optionalKey(VoiceId),
elevenlabs: Schema.optionalKey(VoiceId),
cartesia: Schema.optionalKey(VoiceId),
fish: Schema.optionalKey(VoiceId),
});
export const VoiceProvider = Schema.Literals(["chatgpt", "openai", "composed"]);
export type VoiceProvider = typeof VoiceProvider.Type;
export const VoiceSettings = Schema.Struct({
enabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
provider: VoiceProvider.pipe(Schema.withDecodingDefault(Effect.succeed("chatgpt" as const))),
voice: ChatGptRealtimeVoice.pipe(Schema.withDecodingDefault(Effect.succeed("alloy" as const))),
openaiVoice: Schema.optionalKey(ChatGptRealtimeVoice),
transcriptionProvider: Schema.optionalKey(VoiceTranscriptionProvider),
synthesisProvider: Schema.optionalKey(VoiceApiProvider),
synthesisVoices: Schema.optionalKey(VoiceSynthesisVoices),
}).pipe(Schema.withDecodingDefault(Effect.succeed({})));
export type VoiceSettings = typeof VoiceSettings.Type;
export interface VoiceCapabilities {
readonly realtime: boolean;
readonly transcription: boolean;
readonly synthesis: boolean;
}
/** What each bring-your-own-key provider can do. Only realtime providers support interruption. */
export const VOICE_PROVIDER_CAPABILITIES: Record<VoiceApiProvider, VoiceCapabilities> = {
openai: { realtime: true, transcription: true, synthesis: true },
elevenlabs: { realtime: false, transcription: true, synthesis: true },
cartesia: { realtime: false, transcription: true, synthesis: true },
fish: { realtime: false, transcription: false, synthesis: true },
};
/** API keys a call needs under these settings. ChatGPT calls use the subscription and need none. */
export const voiceRequiredApiProviders = (
settings: VoiceSettings,
): ReadonlyArray<VoiceApiProvider> => {
if (settings.provider === "chatgpt") return [];
if (settings.provider === "openai") return ["openai"];
const transcription = settings.transcriptionProvider ?? "openai";
const synthesis = settings.synthesisProvider ?? "openai";
return transcription === synthesis ? [transcription] : [transcription, synthesis];
};
/** Providers these settings need that have no connected key. Empty means a call can start. */
export const voiceMissingApiProviders = (
settings: VoiceSettings,
connected: ReadonlyArray<VoiceApiProvider>,
): ReadonlyArray<VoiceApiProvider> =>
voiceRequiredApiProviders(settings).filter((provider) => !connected.includes(provider));
export const VoiceCallIdle = Schema.Struct({ status: Schema.Literal("idle") });
export const VoiceCallActive = Schema.Struct({
callId: TrimmedNonEmptyString,
status: Schema.Literals(["starting", "live"]),
botId: BotId,
botName: TrimmedNonEmptyString,
startedAt: IsoDateTime,
});
export const VoiceCallSnapshot = Schema.Union([VoiceCallIdle, VoiceCallActive]);
export type VoiceCallSnapshot = typeof VoiceCallSnapshot.Type;
export const VoiceSessionDescription = Schema.String.check(
Schema.isNonEmpty(),
Schema.isMaxLength(65_536),
);
export const VoiceCallStartInput = Schema.Struct({
botId: BotId,
sdp: Schema.optionalKey(VoiceSessionDescription),
});
export type VoiceCallStartInput = typeof VoiceCallStartInput.Type;
export const VoiceCallHangupInput = Schema.Struct({ callId: TrimmedNonEmptyString });
export type VoiceCallHangupInput = typeof VoiceCallHangupInput.Type;
export const VoiceCallStartResult = Schema.Struct({
call: VoiceCallActive,
answerSdp: Schema.optionalKey(VoiceSessionDescription),
transport: Schema.Literals(["webrtc", "composed"]).pipe(
Schema.withDecodingDefault(Effect.succeed("webrtc" as const)),
),
settings: VoiceSettings,
});
export type VoiceCallStartResult = typeof VoiceCallStartResult.Type;
export const VOICE_AUDIO_MAX_BYTES = 4 * 1024 * 1024;
export const VoiceAudioBase64 = Schema.String.check(
Schema.isNonEmpty(),
Schema.isMaxLength(4 * Math.ceil(VOICE_AUDIO_MAX_BYTES / 3)),
Schema.isPattern(/^[A-Za-z0-9+/]+={0,2}$/),
Schema.makeFilter((value) => value.length % 4 === 0),
);
export const VoiceProviderInput = Schema.Struct({ provider: VoiceApiProvider });
export const VoiceConnectInput = Schema.Struct({
provider: VoiceApiProvider,
apiKey: TrimmedNonEmptyString.check(Schema.isMaxLength(4096)),
});
export const VoiceProviderStatus = Schema.Struct({
provider: VoiceApiProvider,
connected: Schema.Boolean,
/** The provider refused the saved key on its last Test. Cleared when the key changes. */
keyRejected: Schema.optionalKey(Schema.Boolean),
});
export const VoiceProvidersResult = Schema.Struct({ providers: Schema.Array(VoiceProviderStatus) });
export const VoiceListVoicesInput = Schema.Struct({
provider: VoiceApiProvider,
cursor: Schema.optionalKey(VoiceId),
});
export const VoiceListVoicesResult = Schema.Struct({
voices: Schema.Array(
Schema.Struct({ id: VoiceId, name: Schema.String.check(Schema.isMaxLength(256)) }),
).check(Schema.isMaxLength(200)),
nextCursor: Schema.optionalKey(VoiceId),
});
export type VoiceListVoicesResult = typeof VoiceListVoicesResult.Type;
export const VoiceOperationId = TrimmedNonEmptyString.check(Schema.isMaxLength(128));
export const VoiceTranscribeInput = Schema.Struct({
operationId: VoiceOperationId,
callId: Schema.optionalKey(VoiceId),
audioBase64: VoiceAudioBase64,
mimeType: Schema.Literals(["audio/webm", "audio/wav", "audio/mpeg", "audio/mp4", "audio/ogg"]),
});
export type VoiceTranscribeInput = typeof VoiceTranscribeInput.Type;
export const VoiceTranscribeResult = Schema.Struct({
text: Schema.String.check(Schema.isMaxLength(16_000)),
});
export const VoiceSynthesizeInput = Schema.Struct({
operationId: VoiceOperationId,
callId: Schema.optionalKey(VoiceId),
text: Schema.String.check(Schema.isNonEmpty(), Schema.isMaxLength(4_000)),
});
export type VoiceSynthesizeInput = typeof VoiceSynthesizeInput.Type;
export const VoiceSynthesizeResult = Schema.Struct({
audioBase64: VoiceAudioBase64,
mimeType: Schema.Literal("audio/mpeg"),
});
export const VoiceCancelInput = Schema.Struct({ operationId: VoiceOperationId });
export const VoiceCancelResult = Schema.Struct({ cancelled: Schema.Boolean });
export class VoiceCallError extends Schema.TaggedErrorClass<VoiceCallError>()("VoiceCallError", {
reason: Schema.Literals([
"already-active",
"bot-not-found",
"voice-disabled",
"subscription-unavailable",
"upstream-failed",
"call-not-active",
"provider-unavailable",
"provider-in-use",
"invalid-input",
"invalid-voice",
"cancelled",
"busy",
"provider-auth",
"provider-quota",
"network",
]),
message: TrimmedNonEmptyString,
}) {}