The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Register transcription models alongside language models (#25064)
Félix Malfaitbfd3ffc03d81
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/ai-model-kinds.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/ai-model-kinds.const.tsnew file mode 100644index 00000000000..cf65902df9a--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/ai-model-kinds.const.ts@@ -0,0 +1 @@+export const AI_MODEL_KINDS = ['language', 'transcription'] as const;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.tsindex 8fb309edc5f..8cc514ce32c 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts@@ -1,6 +1,6 @@import { Injectable, Logger } from '@nestjs/common';-import { type LanguageModel } from 'ai';+import { type LanguageModel, type TranscriptionModel } from 'ai';import { isNonEmptyString } from '@sniptt/guards';import { type AiSdkPackage } from 'twenty-shared/ai';@@ -16,8 +16,12 @@ import {} from 'src/engine/metadata-modules/ai/ai.exception';import { AiModelPreferencesService } from 'src/engine/metadata-modules/ai/ai-models/services/ai-model-preferences.service';import { ProviderConfigService } from 'src/engine/metadata-modules/ai/ai-models/services/provider-config.service';-import { SdkProviderFactoryService } from 'src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service';+import {+ SdkProviderFactoryService,+ type AiSdkProviderInstance,+} from 'src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service';import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';+import { type AiTranscriptionModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-transcription-model-config.type';import { type AiProviderConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-config.type';import { type AiProviderModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.type';import { type AiProvidersConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-providers-config.type';@@ -26,7 +30,7 @@ import {AUTO_SELECT_FAST_MODEL_ID,AUTO_SELECT_SMART_MODEL_ID,} from 'twenty-shared/constants';-import { isAutoSelectModelId } from 'twenty-shared/utils';+import { isAutoSelectModelId, isDefined } from 'twenty-shared/utils';import { DEFAULT_MAX_OUTPUT_TOKENS } from 'src/engine/metadata-modules/ai/ai-models/types/default-max-output-tokens.const';import { buildCompositeModelId } from 'src/engine/metadata-modules/ai/ai-models/utils/composite-model-id.util';@@ -48,11 +52,24 @@ export interface RegisteredAiModel {modelsDevName?: string;}+export type RegisteredAiTranscriptionModel = {+ modelId: string;+ sdkPackage: AiSdkPackage;+ model: TranscriptionModel;+ providerName: string;+};+@Injectable()export class AiModelRegistryService {private readonly logger = new Logger(AiModelRegistryService.name);private modelRegistry: Map<string, RegisteredAiModel> = new Map();private modelConfigCache: Map<string, AiModelConfig> = new Map();+ // Kept out of modelRegistry and modelConfigCache so transcription models can+ // never surface in the chat model picker or reach token costing.+ private transcriptionRegistry: Map<string, RegisteredAiTranscriptionModel> =+ new Map();+ private transcriptionConfigCache: Map<string, AiTranscriptionModelConfig> =+ new Map();private providerModelDefCache: Map<string,{ providerName: string; modelDef: AiProviderModelConfig }@@ -97,6 +114,8 @@ export class AiModelRegistryService {this.modelRegistry.clear();this.sdkProviderFactory.clearCache();this.modelConfigCache.clear();+ this.transcriptionRegistry.clear();+ this.transcriptionConfigCache.clear();this.providerModelDefCache.clear();const providers = this.providerConfigService.getResolvedProviders({@@ -128,6 +147,18 @@ export class AiModelRegistryService {for (const modelDef of models) {const compositeId = buildCompositeModelId(providerKey, modelDef.name);+ if (modelDef.kind === 'transcription') {+ this.registerTranscriptionModel({+ compositeId,+ providerKey,+ config,+ modelDef,+ sdkInstance,+ });++ continue;+ }+this.modelConfigCache.set(compositeId,this.toAiModelConfig(compositeId, config, modelDef),@@ -152,6 +183,99 @@ export class AiModelRegistryService {}}+ private registerTranscriptionModel({+ compositeId,+ providerKey,+ config,+ modelDef,+ sdkInstance,+ }: {+ compositeId: string;+ providerKey: string;+ config: AiProviderConfig;+ modelDef: AiProviderModelConfig;+ sdkInstance: AiSdkProviderInstance | undefined;+ }): void {+ // An omitted price bills nothing while the provider still charges, so the+ // model is refused rather than run for free. An explicit 0 is allowed.+ if (!isDefined(modelDef.costPerMinute)) {+ this.logger.error(+ `Skipping transcription model "${compositeId}": costPerMinute is required`,+ );++ return;+ }++ this.transcriptionConfigCache.set(compositeId, {+ modelId: compositeId,+ sdkPackage: config.npm,+ label: modelDef.label,+ description: modelDef.description ?? compositeId,+ dataResidency: config.dataResidency,+ costPerMinute: modelDef.costPerMinute,+ isDeprecated: modelDef.isDeprecated,+ });++ if (!sdkInstance) {+ return;+ }++ const createTranscriptionModel = sdkInstance.createTranscriptionModel;++ if (!isDefined(createTranscriptionModel)) {+ this.logger.warn(+ `Skipping transcription model "${compositeId}": ${config.npm} exposes no transcription API`,+ );++ return;+ }++ this.transcriptionRegistry.set(compositeId, {+ modelId: compositeId,+ sdkPackage: config.npm,+ model: createTranscriptionModel(modelDef.name),+ providerName: providerKey,+ });+ }++ getTranscriptionModel(+ modelId: string,+ ): RegisteredAiTranscriptionModel | undefined {+ this.ensureFresh();++ return this.transcriptionRegistry.get(modelId);+ }++ getAvailableTranscriptionModels(): RegisteredAiTranscriptionModel[] {+ this.ensureFresh();++ return Array.from(this.transcriptionRegistry.values());+ }++ // Registration order follows the provider config, so the first entry is the+ // one an operator listed first.+ getDefaultTranscriptionModel(): RegisteredAiTranscriptionModel | undefined {+ return this.getAvailableTranscriptionModels().find(+ (model) =>+ this.getTranscriptionModelConfig(model.modelId)?.isDeprecated !== true,+ );+ }++ getTranscriptionModelConfig(+ modelId: string,+ ): AiTranscriptionModelConfig | undefined {+ this.ensureFresh();++ return this.transcriptionConfigCache.get(modelId);+ }++ // Deliberately the same rule as getDefaultTranscriptionModel: a registry+ // holding only deprecated models would otherwise advertise cloud dictation+ // that every request without an explicit model id then fails to resolve.+ hasTranscriptionModel(): boolean {+ return isDefined(this.getDefaultTranscriptionModel());+ }+private toAiModelConfig(compositeId: string,providerConfig: AiProviderConfig,diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.tsindex 85e6ceeb0ea..dbd8ab282f6 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/sdk-provider-factory.service.ts@@ -13,8 +13,10 @@ import {wrapLanguageModel,type LanguageModel,type LanguageModelMiddleware,+ type TranscriptionModel,} from 'ai';import { type AiSdkPackage } from 'twenty-shared/ai';+import { isDefined } from 'twenty-shared/utils';import {AI_SDK_ANTHROPIC,@@ -28,9 +30,12 @@ import {} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';import { sanitizeGeminiToolResultRefsMiddleware } from 'src/engine/metadata-modules/ai/ai-models/middleware/sanitize-gemini-tool-result-refs.middleware';import { type AiProviderConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-config.type';+import { getTranscriptionModelFactory } from 'src/engine/metadata-modules/ai/ai-models/utils/get-transcription-model-factory.util';export type AiSdkProviderInstance = {createModel: (modelId: string) => LanguageModel;+ // Absent on providers with no speech-to-text API.+ createTranscriptionModel?: (modelId: string) => TranscriptionModel;rawProvider: unknown;sdkPackage: AiSdkPackage;};@@ -88,6 +93,21 @@ export class SdkProviderFactoryService {this.providerInstances.clear();}+ private toProviderInstance(+ provider: unknown,+ sdkPackage: AiSdkPackage,+ createModel: (modelId: string) => LanguageModel,+ ): AiSdkProviderInstance {+ const createTranscriptionModel = getTranscriptionModelFactory(provider);++ return {+ createModel,+ ...(isDefined(createTranscriptionModel) && { createTranscriptionModel }),+ rawProvider: provider,+ sdkPackage,+ };+ }+private buildProviderInstance(config: AiProviderConfig,): AiSdkProviderInstance {@@ -125,17 +145,13 @@ export class SdkProviderFactoryService {...(config.baseUrl && { baseURL: config.baseUrl }),});- return {- createModel: (modelId: string) => {- const model = (provider as CallableFunction)(modelId);+ return this.toProviderInstance(provider, config.npm, (modelId: string) => {+ const model = (provider as CallableFunction)(modelId);- return options?.middleware- ? wrapLanguageModel({ model, middleware: options.middleware })- : model;- },- rawProvider: provider,- sdkPackage: config.npm,- };+ return options?.middleware+ ? wrapLanguageModel({ model, middleware: options.middleware })+ : model;+ });}private buildXaiProvider(config: AiProviderConfig): AiSdkProviderInstance {@@ -144,11 +160,9 @@ export class SdkProviderFactoryService {...(config.baseUrl && { baseURL: config.baseUrl }),});- return {- createModel: (modelId: string) => provider.responses(modelId),- rawProvider: provider,- sdkPackage: AI_SDK_XAI,- };+ return this.toProviderInstance(provider, AI_SDK_XAI, (modelId: string) =>+ provider.responses(modelId),+ );}private buildBedrockProvider(@@ -182,11 +196,11 @@ export class SdkProviderFactoryService {}),});- return {- createModel: (modelId: string) => provider(modelId),- rawProvider: provider,- sdkPackage: AI_SDK_BEDROCK,- };+ return this.toProviderInstance(+ provider,+ AI_SDK_BEDROCK,+ (modelId: string) => provider(modelId),+ );}private buildOpenAiCompatibleProvider(@@ -202,11 +216,11 @@ export class SdkProviderFactoryService {...(config.apiKey && { apiKey: config.apiKey }),});- return {- createModel: (modelId: string) => provider(modelId),- rawProvider: provider,- sdkPackage: AI_SDK_OPENAI_COMPATIBLE,- };+ return this.toProviderInstance(+ provider,+ AI_SDK_OPENAI_COMPATIBLE,+ (modelId: string) => provider(modelId),+ );}private buildAzureProvider(config: AiProviderConfig): AiSdkProviderInstance {@@ -219,10 +233,8 @@ export class SdkProviderFactoryService {...(config.apiKey && { apiKey: config.apiKey }),});- return {- createModel: (modelId: string) => provider(modelId),- rawProvider: provider,- sdkPackage: AI_SDK_AZURE,- };+ return this.toProviderInstance(provider, AI_SDK_AZURE, (modelId: string) =>+ provider(modelId),+ );}}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.tsindex 465e9856857..d220d46e6b4 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts@@ -1,21 +1,35 @@import { z } from 'zod';+import { AI_MODEL_KINDS } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-model-kinds.const';import { ModelFamily } from 'src/engine/metadata-modules/ai/ai-models/types/model-family.enum';import { longContextCostSchema } from 'src/engine/metadata-modules/ai/ai-models/types/long-context-cost.schema';-export const aiProviderModelConfigSchema = z.object({- name: z.string(),- label: z.string(),- description: z.string().optional(),- modelFamily: z.nativeEnum(ModelFamily).optional(),- inputCostPerMillionTokens: z.number().optional(),- outputCostPerMillionTokens: z.number().optional(),- cachedInputCostPerMillionTokens: z.number().optional(),- cacheCreationCostPerMillionTokens: z.number().optional(),- longContextCost: longContextCostSchema.optional(),- contextWindowTokens: z.number().int().positive().optional(),- maxOutputTokens: z.number().int().positive().optional(),- modalities: z.array(z.string()).optional(),- supportsReasoning: z.boolean().optional(),- isDeprecated: z.boolean().optional(),-});+export const aiProviderModelConfigSchema = z+ .object({+ name: z.string(),+ label: z.string(),+ kind: z.enum(AI_MODEL_KINDS).optional(),+ description: z.string().optional(),+ modelFamily: z.nativeEnum(ModelFamily).optional(),+ costPerMinute: z.number().nonnegative().optional(),+ inputCostPerMillionTokens: z.number().optional(),+ outputCostPerMillionTokens: z.number().optional(),+ cachedInputCostPerMillionTokens: z.number().optional(),+ cacheCreationCostPerMillionTokens: z.number().optional(),+ longContextCost: longContextCostSchema.optional(),+ contextWindowTokens: z.number().int().positive().optional(),+ maxOutputTokens: z.number().int().positive().optional(),+ modalities: z.array(z.string()).optional(),+ supportsReasoning: z.boolean().optional(),+ isDeprecated: z.boolean().optional(),+ })+ .refine(+ (model) =>+ model.kind !== 'transcription' || model.costPerMinute !== undefined,+ {+ // An omitted price bills nothing while the provider still charges, so a+ // free model has to say so with an explicit 0.+ message: 'costPerMinute is required for transcription models',+ path: ['costPerMinute'],+ },+ );diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-transcription-model-config.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-transcription-model-config.type.tsnew file mode 100644index 00000000000..14b82b79c4e--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-transcription-model-config.type.ts@@ -0,0 +1,13 @@+import { type AiSdkPackage, type DataResidency } from 'twenty-shared/ai';++// Separate from AiModelConfig so computeCostBreakdown is never reachable with a+// model that has no token semantics.+export type AiTranscriptionModelConfig = {+ modelId: string;+ sdkPackage: AiSdkPackage;+ label: string;+ description: string;+ dataResidency?: DataResidency;+ costPerMinute: number;+ isDeprecated?: boolean;+};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-transcription-model-factory.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-transcription-model-factory.util.spec.tsnew file mode 100644index 00000000000..d2b445d9677--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-transcription-model-factory.util.spec.ts@@ -0,0 +1,37 @@+import { getTranscriptionModelFactory } from 'src/engine/metadata-modules/ai/ai-models/utils/get-transcription-model-factory.util';++describe('getTranscriptionModelFactory', () => {+ it('returns a factory for a provider exposing transcription', () => {+ const model = { modelId: 'whisper-1' };+ const provider = { transcription: jest.fn().mockReturnValue(model) };++ const factory = getTranscriptionModelFactory(provider);++ expect(factory).toBeDefined();+ expect(factory?.('whisper-1')).toBe(model);+ expect(provider.transcription).toHaveBeenCalledTimes(1);+ expect(provider.transcription).toHaveBeenCalledWith('whisper-1');+ });++ it('keeps the provider as the receiver so provider-bound state survives', () => {+ const provider = {+ baseUrl: 'https://example.invalid',+ transcription(this: { baseUrl: string }, modelId: string) {+ return `${this.baseUrl}/${modelId}`;+ },+ };++ const factory = getTranscriptionModelFactory(provider);++ expect(factory?.('whisper-1')).toBe('https://example.invalid/whisper-1');+ });++ it.each([+ ['a provider without the method', { languageModel: jest.fn() }],+ ['a non-callable transcription property', { transcription: 'nope' }],+ ['null', null],+ ['undefined', undefined],+ ])('returns undefined for %s', (_label, provider) => {+ expect(getTranscriptionModelFactory(provider)).toBeUndefined();+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-transcription-model-factory.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-transcription-model-factory.util.tsnew file mode 100644index 00000000000..0cdb2a10785--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-transcription-model-factory.util.ts@@ -0,0 +1,21 @@+import { type TranscriptionModel } from 'ai';++type TranscriptionCapableProvider = {+ transcription: (modelId: string) => TranscriptionModel;+};++const hasTranscriptionFactory = (+ provider: unknown,+): provider is TranscriptionCapableProvider =>+ typeof (provider as Partial<TranscriptionCapableProvider>)?.transcription ===+ 'function';++export const getTranscriptionModelFactory = (+ provider: unknown,+): ((modelId: string) => TranscriptionModel) | undefined => {+ if (!hasTranscriptionFactory(provider)) {+ return undefined;+ }++ return (modelId: string) => provider.transcription(modelId);+};