Skip to content

The open alternative to Salesforce, designed for AI.

Register transcription models alongside language models (#25064)

Félix Malfaitbfd3ffc03d81
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/ai-model-kinds.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/ai-model-kinds.const.ts
new file mode 100644
index 00000000000..cf65902df9a
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/ai-model-kinds.const.ts
@@ -0,0 +1 @@
+export const AI_MODEL_KINDS = ['language', 'transcription'] as const;
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts
index 8fb309edc5f..8cc514ce32c 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts
@@ -1,6 +1,6 @@
import { Injectable, Logger } from '@nestjs/common';
-import { type LanguageModel } from 'ai';
+import { type LanguageModel, type TranscriptionModel } from 'ai';
import { isNonEmptyString } from '@sniptt/guards';
import { type AiSdkPackage } from 'twenty-shared/ai';
@@ -16,8 +16,12 @@ import {
} from 'src/engine/metadata-modules/ai/ai.exception';
import { AiModelPreferencesService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-preferences.service';
import { ProviderConfigService } from 'src/engine/metadata-modules/ai/ai-models/services/provider-config.service';
-import { SdkProviderFactoryService } from 'src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service';
+import {
+ SdkProviderFactoryService,
+ type AiSdkProviderInstance,
+} from 'src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service';
import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';
+import { type AiTranscriptionModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-transcription-model-config.type';
import { type AiProviderConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-config.type';
import { type AiProviderModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.type';
import { type AiProvidersConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-providers-config.type';
@@ -26,7 +30,7 @@ import {
AUTO_SELECT_FAST_MODEL_ID,
AUTO_SELECT_SMART_MODEL_ID,
} from 'twenty-shared/constants';
-import { isAutoSelectModelId } from 'twenty-shared/utils';
+import { isAutoSelectModelId, isDefined } from 'twenty-shared/utils';
import { DEFAULT_MAX_OUTPUT_TOKENS } from 'src/engine/metadata-modules/ai/ai-models/types/default-max-output-tokens.const';
import { buildCompositeModelId } from 'src/engine/metadata-modules/ai/ai-models/utils/composite-model-id.util';
@@ -48,11 +52,24 @@ export interface RegisteredAiModel {
modelsDevName?: string;
}
+export type RegisteredAiTranscriptionModel = {
+ modelId: string;
+ sdkPackage: AiSdkPackage;
+ model: TranscriptionModel;
+ providerName: string;
+};
+
@Injectable()
export class AiModelRegistryService {
private readonly logger = new Logger(AiModelRegistryService.name);
private modelRegistry: Map<string, RegisteredAiModel> = new Map();
private modelConfigCache: Map<string, AiModelConfig> = new Map();
+ // Kept out of modelRegistry and modelConfigCache so transcription models can
+ // never surface in the chat model picker or reach token costing.
+ private transcriptionRegistry: Map<string, RegisteredAiTranscriptionModel> =
+ new Map();
+ private transcriptionConfigCache: Map<string, AiTranscriptionModelConfig> =
+ new Map();
private providerModelDefCache: Map<
string,
{ providerName: string; modelDef: AiProviderModelConfig }
@@ -97,6 +114,8 @@ export class AiModelRegistryService {
this.modelRegistry.clear();
this.sdkProviderFactory.clearCache();
this.modelConfigCache.clear();
+ this.transcriptionRegistry.clear();
+ this.transcriptionConfigCache.clear();
this.providerModelDefCache.clear();
const providers = this.providerConfigService.getResolvedProviders({
@@ -128,6 +147,18 @@ export class AiModelRegistryService {
for (const modelDef of models) {
const compositeId = buildCompositeModelId(providerKey, modelDef.name);
+ if (modelDef.kind === 'transcription') {
+ this.registerTranscriptionModel({
+ compositeId,
+ providerKey,
+ config,
+ modelDef,
+ sdkInstance,
+ });
+
+ continue;
+ }
+
this.modelConfigCache.set(
compositeId,
this.toAiModelConfig(compositeId, config, modelDef),
@@ -152,6 +183,99 @@ export class AiModelRegistryService {
}
}
+ private registerTranscriptionModel({
+ compositeId,
+ providerKey,
+ config,
+ modelDef,
+ sdkInstance,
+ }: {
+ compositeId: string;
+ providerKey: string;
+ config: AiProviderConfig;
+ modelDef: AiProviderModelConfig;
+ sdkInstance: AiSdkProviderInstance | undefined;
+ }): void {
+ // An omitted price bills nothing while the provider still charges, so the
+ // model is refused rather than run for free. An explicit 0 is allowed.
+ if (!isDefined(modelDef.costPerMinute)) {
+ this.logger.error(
+ `Skipping transcription model "${compositeId}": costPerMinute is required`,
+ );
+
+ return;
+ }
+
+ this.transcriptionConfigCache.set(compositeId, {
+ modelId: compositeId,
+ sdkPackage: config.npm,
+ label: modelDef.label,
+ description: modelDef.description ?? compositeId,
+ dataResidency: config.dataResidency,
+ costPerMinute: modelDef.costPerMinute,
+ isDeprecated: modelDef.isDeprecated,
+ });
+
+ if (!sdkInstance) {
+ return;
+ }
+
+ const createTranscriptionModel = sdkInstance.createTranscriptionModel;
+
+ if (!isDefined(createTranscriptionModel)) {
+ this.logger.warn(
+ `Skipping transcription model "${compositeId}": ${config.npm} exposes no transcription API`,
+ );
+
+ return;
+ }
+
+ this.transcriptionRegistry.set(compositeId, {
+ modelId: compositeId,
+ sdkPackage: config.npm,
+ model: createTranscriptionModel(modelDef.name),
+ providerName: providerKey,
+ });
+ }
+
+ getTranscriptionModel(
+ modelId: string,
+ ): RegisteredAiTranscriptionModel | undefined {
+ this.ensureFresh();
+
+ return this.transcriptionRegistry.get(modelId);
+ }
+
+ getAvailableTranscriptionModels(): RegisteredAiTranscriptionModel[] {
+ this.ensureFresh();
+
+ return Array.from(this.transcriptionRegistry.values());
+ }
+
+ // Registration order follows the provider config, so the first entry is the
+ // one an operator listed first.
+ getDefaultTranscriptionModel(): RegisteredAiTranscriptionModel | undefined {
+ return this.getAvailableTranscriptionModels().find(
+ (model) =>
+ this.getTranscriptionModelConfig(model.modelId)?.isDeprecated !== true,
+ );
+ }
+
+ getTranscriptionModelConfig(
+ modelId: string,
+ ): AiTranscriptionModelConfig | undefined {
+ this.ensureFresh();
+
+ return this.transcriptionConfigCache.get(modelId);
+ }
+
+ // Deliberately the same rule as getDefaultTranscriptionModel: a registry
+ // holding only deprecated models would otherwise advertise cloud dictation
+ // that every request without an explicit model id then fails to resolve.
+ hasTranscriptionModel(): boolean {
+ return isDefined(this.getDefaultTranscriptionModel());
+ }
+
private toAiModelConfig(
compositeId: string,
providerConfig: AiProviderConfig,
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts
index 85e6ceeb0ea..dbd8ab282f6 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts
@@ -13,8 +13,10 @@ import {
wrapLanguageModel,
type LanguageModel,
type LanguageModelMiddleware,
+ type TranscriptionModel,
} from 'ai';
import { type AiSdkPackage } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
import {
AI_SDK_ANTHROPIC,
@@ -28,9 +30,12 @@ import {
} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';
import { sanitizeGeminiToolResultRefsMiddleware } from 'src/engine/metadata-modules/ai/ai-models/middleware/sanitize-gemini-tool-result-refs.middleware';
import { type AiProviderConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-config.type';
+import { getTranscriptionModelFactory } from 'src/engine/metadata-modules/ai/ai-models/utils/get-transcription-model-factory.util';
export type AiSdkProviderInstance = {
createModel: (modelId: string) => LanguageModel;
+ // Absent on providers with no speech-to-text API.
+ createTranscriptionModel?: (modelId: string) => TranscriptionModel;
rawProvider: unknown;
sdkPackage: AiSdkPackage;
};
@@ -88,6 +93,21 @@ export class SdkProviderFactoryService {
this.providerInstances.clear();
}
+ private toProviderInstance(
+ provider: unknown,
+ sdkPackage: AiSdkPackage,
+ createModel: (modelId: string) => LanguageModel,
+ ): AiSdkProviderInstance {
+ const createTranscriptionModel = getTranscriptionModelFactory(provider);
+
+ return {
+ createModel,
+ ...(isDefined(createTranscriptionModel) && { createTranscriptionModel }),
+ rawProvider: provider,
+ sdkPackage,
+ };
+ }
+
private buildProviderInstance(
config: AiProviderConfig,
): AiSdkProviderInstance {
@@ -125,17 +145,13 @@ export class SdkProviderFactoryService {
...(config.baseUrl && { baseURL: config.baseUrl }),
});
- return {
- createModel: (modelId: string) => {
- const model = (provider as CallableFunction)(modelId);
+ return this.toProviderInstance(provider, config.npm, (modelId: string) => {
+ const model = (provider as CallableFunction)(modelId);
- return options?.middleware
- ? wrapLanguageModel({ model, middleware: options.middleware })
- : model;
- },
- rawProvider: provider,
- sdkPackage: config.npm,
- };
+ return options?.middleware
+ ? wrapLanguageModel({ model, middleware: options.middleware })
+ : model;
+ });
}
private buildXaiProvider(config: AiProviderConfig): AiSdkProviderInstance {
@@ -144,11 +160,9 @@ export class SdkProviderFactoryService {
...(config.baseUrl && { baseURL: config.baseUrl }),
});
- return {
- createModel: (modelId: string) => provider.responses(modelId),
- rawProvider: provider,
- sdkPackage: AI_SDK_XAI,
- };
+ return this.toProviderInstance(provider, AI_SDK_XAI, (modelId: string) =>
+ provider.responses(modelId),
+ );
}
private buildBedrockProvider(
@@ -182,11 +196,11 @@ export class SdkProviderFactoryService {
}),
});
- return {
- createModel: (modelId: string) => provider(modelId),
- rawProvider: provider,
- sdkPackage: AI_SDK_BEDROCK,
- };
+ return this.toProviderInstance(
+ provider,
+ AI_SDK_BEDROCK,
+ (modelId: string) => provider(modelId),
+ );
}
private buildOpenAiCompatibleProvider(
@@ -202,11 +216,11 @@ export class SdkProviderFactoryService {
...(config.apiKey && { apiKey: config.apiKey }),
});
- return {
- createModel: (modelId: string) => provider(modelId),
- rawProvider: provider,
- sdkPackage: AI_SDK_OPENAI_COMPATIBLE,
- };
+ return this.toProviderInstance(
+ provider,
+ AI_SDK_OPENAI_COMPATIBLE,
+ (modelId: string) => provider(modelId),
+ );
}
private buildAzureProvider(config: AiProviderConfig): AiSdkProviderInstance {
@@ -219,10 +233,8 @@ export class SdkProviderFactoryService {
...(config.apiKey && { apiKey: config.apiKey }),
});
- return {
- createModel: (modelId: string) => provider(modelId),
- rawProvider: provider,
- sdkPackage: AI_SDK_AZURE,
- };
+ return this.toProviderInstance(provider, AI_SDK_AZURE, (modelId: string) =>
+ provider(modelId),
+ );
}
}
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts
index 465e9856857..d220d46e6b4 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts
@@ -1,21 +1,35 @@
import { z } from 'zod';
+import { AI_MODEL_KINDS } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-model-kinds.const';
import { ModelFamily } from 'src/engine/metadata-modules/ai/ai-models/types/model-family.enum';
import { longContextCostSchema } from 'src/engine/metadata-modules/ai/ai-models/types/long-context-cost.schema';
-export const aiProviderModelConfigSchema = z.object({
- name: z.string(),
- label: z.string(),
- description: z.string().optional(),
- modelFamily: z.nativeEnum(ModelFamily).optional(),
- inputCostPerMillionTokens: z.number().optional(),
- outputCostPerMillionTokens: z.number().optional(),
- cachedInputCostPerMillionTokens: z.number().optional(),
- cacheCreationCostPerMillionTokens: z.number().optional(),
- longContextCost: longContextCostSchema.optional(),
- contextWindowTokens: z.number().int().positive().optional(),
- maxOutputTokens: z.number().int().positive().optional(),
- modalities: z.array(z.string()).optional(),
- supportsReasoning: z.boolean().optional(),
- isDeprecated: z.boolean().optional(),
-});
+export const aiProviderModelConfigSchema = z
+ .object({
+ name: z.string(),
+ label: z.string(),
+ kind: z.enum(AI_MODEL_KINDS).optional(),
+ description: z.string().optional(),
+ modelFamily: z.nativeEnum(ModelFamily).optional(),
+ costPerMinute: z.number().nonnegative().optional(),
+ inputCostPerMillionTokens: z.number().optional(),
+ outputCostPerMillionTokens: z.number().optional(),
+ cachedInputCostPerMillionTokens: z.number().optional(),
+ cacheCreationCostPerMillionTokens: z.number().optional(),
+ longContextCost: longContextCostSchema.optional(),
+ contextWindowTokens: z.number().int().positive().optional(),
+ maxOutputTokens: z.number().int().positive().optional(),
+ modalities: z.array(z.string()).optional(),
+ supportsReasoning: z.boolean().optional(),
+ isDeprecated: z.boolean().optional(),
+ })
+ .refine(
+ (model) =>
+ model.kind !== 'transcription' || model.costPerMinute !== undefined,
+ {
+ // An omitted price bills nothing while the provider still charges, so a
+ // free model has to say so with an explicit 0.
+ message: 'costPerMinute is required for transcription models',
+ path: ['costPerMinute'],
+ },
+ );
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-transcription-model-config.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-transcription-model-config.type.ts
new file mode 100644
index 00000000000..14b82b79c4e
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-transcription-model-config.type.ts
@@ -0,0 +1,13 @@
+import { type AiSdkPackage, type DataResidency } from 'twenty-shared/ai';
+
+// Separate from AiModelConfig so computeCostBreakdown is never reachable with a
+// model that has no token semantics.
+export type AiTranscriptionModelConfig = {
+ modelId: string;
+ sdkPackage: AiSdkPackage;
+ label: string;
+ description: string;
+ dataResidency?: DataResidency;
+ costPerMinute: number;
+ isDeprecated?: boolean;
+};
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-transcription-model-factory.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-transcription-model-factory.util.spec.ts
new file mode 100644
index 00000000000..d2b445d9677
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-transcription-model-factory.util.spec.ts
@@ -0,0 +1,37 @@
+import { getTranscriptionModelFactory } from 'src/engine/metadata-modules/ai/ai-models/utils/get-transcription-model-factory.util';
+
+describe('getTranscriptionModelFactory', () => {
+ it('returns a factory for a provider exposing transcription', () => {
+ const model = { modelId: 'whisper-1' };
+ const provider = { transcription: jest.fn().mockReturnValue(model) };
+
+ const factory = getTranscriptionModelFactory(provider);
+
+ expect(factory).toBeDefined();
+ expect(factory?.('whisper-1')).toBe(model);
+ expect(provider.transcription).toHaveBeenCalledTimes(1);
+ expect(provider.transcription).toHaveBeenCalledWith('whisper-1');
+ });
+
+ it('keeps the provider as the receiver so provider-bound state survives', () => {
+ const provider = {
+ baseUrl: 'https://example.invalid',
+ transcription(this: { baseUrl: string }, modelId: string) {
+ return `${this.baseUrl}/${modelId}`;
+ },
+ };
+
+ const factory = getTranscriptionModelFactory(provider);
+
+ expect(factory?.('whisper-1')).toBe('https://example.invalid/whisper-1');
+ });
+
+ it.each([
+ ['a provider without the method', { languageModel: jest.fn() }],
+ ['a non-callable transcription property', { transcription: 'nope' }],
+ ['null', null],
+ ['undefined', undefined],
+ ])('returns undefined for %s', (_label, provider) => {
+ expect(getTranscriptionModelFactory(provider)).toBeUndefined();
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-transcription-model-factory.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-transcription-model-factory.util.ts
new file mode 100644
index 00000000000..0cdb2a10785
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-transcription-model-factory.util.ts
@@ -0,0 +1,21 @@
+import { type TranscriptionModel } from 'ai';
+
+type TranscriptionCapableProvider = {
+ transcription: (modelId: string) => TranscriptionModel;
+};
+
+const hasTranscriptionFactory = (
+ provider: unknown,
+): provider is TranscriptionCapableProvider =>
+ typeof (provider as Partial<TranscriptionCapableProvider>)?.transcription ===
+ 'function';
+
+export const getTranscriptionModelFactory = (
+ provider: unknown,
+): ((modelId: string) => TranscriptionModel) | undefined => {
+ if (!hasTranscriptionFactory(provider)) {
+ return undefined;
+ }
+
+ return (modelId: string) => provider.transcription(modelId);
+};