The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Score effort variants from the benchmark rows taken at their own effort (#25730)
Félix Malfait9989ec6bca57
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/enrich-catalog.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/enrich-catalog.spec.tsindex 9a36bf485d1..f9c9c3eb41c 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/enrich-catalog.spec.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/enrich-catalog.spec.ts@@ -77,4 +77,61 @@ describe('enrichCatalog', () => {'isDeprecated',]);});++ it('embeds per-effort readings in the catalog and mirrors them, with aliases, into the overlay', () => {+ const catalog = catalogOf('gpt-x');++ catalog.openai.models[0].efforts = ['low', 'high'];++ const overlay = enrich(+ catalog,+ new Map([+ [+ 'gptx',+ { intelligenceIndex: 41, effort: 'high', aliases: ['GPT X (high)'] },+ ],+ [+ 'gptx@high',+ { intelligenceIndex: 41, effort: 'high', aliases: ['GPT X (high)'] },+ ],+ [+ 'gptx@low',+ { intelligenceIndex: 30, effort: 'low', aliases: ['GPT X (low)'] },+ ],+ ]),+ );++ expect(catalog.openai.models[0].benchmark?.effort).toBe('high');+ expect(catalog.openai.models[0].benchmarkByEffort).toEqual({+ low: { intelligenceIndex: 30, effort: 'low', measuredAt: MEASURED_AT },+ high: { intelligenceIndex: 41, effort: 'high', measuredAt: MEASURED_AT },+ });+ expect(overlay['gpt-x'].benchmarkByEffort?.low).toEqual({+ intelligenceIndex: 30,+ effort: 'low',+ measuredAt: MEASURED_AT,+ aliases: ['GPT X (low)'],+ });+ expect(Object.keys(catalog.openai.models[0])).toEqual([+ 'name',+ 'label',+ 'efforts',+ 'benchmark',+ 'benchmarkByEffort',+ ]);+ });++ it('writes no effort map for a model whose declared efforts have no rows', () => {+ const catalog = catalogOf('gpt-x');++ catalog.openai.models[0].efforts = ['low', 'high'];++ const overlay = enrich(+ catalog,+ new Map([['gptx', { intelligenceIndex: 41, aliases: ['gpt-x'] }]]),+ );++ expect(catalog.openai.models[0].benchmarkByEffort).toBeUndefined();+ expect(overlay['gpt-x'].benchmarkByEffort).toBeUndefined();+ });});diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.tsindex e90061e4cfe..49b0aa4855e 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts@@ -142,4 +142,66 @@ describe('fetchArtificialAnalysisBenchmarks', () => {expect(record?.intelligenceIndex).toBeUndefined();expect(record?.outputTokensPerSecond).toBe(92.1);});++ it('files a row under its effort as well as under the bare model name', async () => {+ respondWith([+ {+ slug: 'gpt-5-6-sol',+ name: 'GPT-5.6 Sol (max)',+ evaluations: { artificial_analysis_intelligence_index: 47.1 },+ },+ {+ slug: 'gpt-5-6-sol',+ name: 'GPT-5.6 Sol (low)',+ evaluations: { artificial_analysis_intelligence_index: 30.2 },+ cost_per_task: { total_cost: 0.08 },+ },+ ]);++ const index = await fetchArtificialAnalysisBenchmarks('key');++ // The bare key still carries the ceiling, so nothing a consumer reads today+ // changes.+ expect(index.get('gpt56sol')?.intelligenceIndex).toBe(47.1);+ expect(index.get('gpt56sol')?.effort).toBe('max');+ expect(index.get('gpt56sol@max')?.intelligenceIndex).toBe(47.1);+ expect(index.get('gpt56sol@low')?.intelligenceIndex).toBe(30.2);+ expect(index.get('gpt56sol@low')?.costPerTask).toBe(0.08);+ expect(index.get('gpt56sol@low')?.effort).toBe('low');+ });++ it('keeps the better-measured row when two rows share an effort', async () => {+ respondWith([+ {+ slug: 'grok-4-6',+ name: 'Grok 4.6 (high)',+ evaluations: { artificial_analysis_intelligence_index: 44.4 },+ },+ {+ slug: 'grok-4-6',+ name: 'Grok 4.6 (high)',+ evaluations: { artificial_analysis_intelligence_index: 44.4 },+ median_output_tokens_per_second: 71,+ },+ ]);++ const index = await fetchArtificialAnalysisBenchmarks('key');++ expect(index.get('grok46@high')?.outputTokensPerSecond).toBe(71);+ });++ it('files a row naming no effort under the bare name only', async () => {+ respondWith([+ {+ slug: 'gemini-3-1-pro-preview',+ name: 'Gemini 3.1 Pro Preview',+ evaluations: { artificial_analysis_intelligence_index: 30.4 },+ },+ ]);++ const index = await fetchArtificialAnalysisBenchmarks('key');++ expect(index.get('gemini31propreview')?.effort).toBeUndefined();+ expect([...index.keys()].some((key) => key.includes('@'))).toBe(false);+ });});diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.tsindex 5afeb2f3527..39c07ea94a9 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts@@ -217,4 +217,123 @@ describe('matchBenchmarks', () => {'mistralai/mistral-large-2512',]);});++ it('reads each declared effort from the row taken at that effort', () => {+ const result = matchBenchmarks({+ modelName: 'mistral-large-2512',+ siblingModels: MISTRAL_MODELS,+ benchmarkIndex: indexOf({+ mistrallarge2512: {+ intelligenceIndex: 41,+ effort: 'high',+ aliases: ['Mistral Large 3 (high)'],+ },+ 'mistrallarge2512@high': {+ intelligenceIndex: 41,+ effort: 'high',+ aliases: ['Mistral Large 3 (high)'],+ },+ 'mistrallarge2512@none': {+ intelligenceIndex: 22,+ costPerTask: 0.1,+ effort: 'none',+ aliases: ['Mistral Large 3 (Non-reasoning)'],+ },+ }),+ measuredAt: MEASURED_AT,+ efforts: ['none', 'high'],+ });++ expect(result?.benchmark.effort).toBe('high');+ expect(result?.benchmarkByEffort?.none).toEqual({+ intelligenceIndex: 22,+ costPerTask: 0.1,+ effort: 'none',+ measuredAt: MEASURED_AT,+ aliases: ['Mistral Large 3 (Non-reasoning)'],+ });+ expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);+ });++ it('leaves a declared effort without its own row blank rather than lending it the ceiling', () => {+ const result = matchBenchmarks({+ modelName: 'mistral-large-2512',+ siblingModels: MISTRAL_MODELS,+ benchmarkIndex: indexOf({+ mistrallarge2512: {+ intelligenceIndex: 41,+ effort: 'high',+ aliases: [],+ },+ 'mistrallarge2512@high': {+ intelligenceIndex: 41,+ effort: 'high',+ aliases: [],+ },+ }),+ measuredAt: MEASURED_AT,+ efforts: ['none', 'high'],+ });++ expect(result?.benchmarkByEffort?.none).toBeUndefined();+ expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);+ });++ it('carries no effort map at all when no declared effort has a row', () => {+ const result = matchBenchmarks({+ modelName: 'mistral-large-2512',+ siblingModels: MISTRAL_MODELS,+ benchmarkIndex: indexOf({+ mistrallarge2512: { intelligenceIndex: 41, aliases: [] },+ }),+ measuredAt: MEASURED_AT,+ efforts: ['none', 'high'],+ });++ expect(result?.benchmark.intelligenceIndex).toBe(41);+ expect(result?.benchmarkByEffort).toBeUndefined();+ });++ it('resolves a rolling alias to its release for per-effort rows too', () => {+ const result = matchBenchmarks({+ modelName: 'mistral-large-latest',+ siblingModels: MISTRAL_MODELS,+ benchmarkIndex: indexOf({+ mistrallarge2512: { intelligenceIndex: 41, aliases: [] },+ 'mistrallarge2512@high': {+ intelligenceIndex: 41,+ effort: 'high',+ aliases: [],+ },+ 'mistrallarge@high': {+ intelligenceIndex: 20,+ effort: 'high',+ aliases: [],+ },+ }),+ measuredAt: MEASURED_AT,+ efforts: ['high'],+ });++ expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);+ });++ it('labels a per-effort reading with the key it was filed under', () => {+ const result = matchBenchmarks({+ modelName: 'mistral-large-2512',+ siblingModels: MISTRAL_MODELS,+ benchmarkIndex: indexOf({+ mistrallarge2512: { intelligenceIndex: 41, aliases: [] },+ 'mistrallarge2512@low': {+ intelligenceIndex: 30,+ effort: 'high',+ aliases: [],+ },+ }),+ measuredAt: MEASURED_AT,+ efforts: ['low'],+ });++ expect(result?.benchmarkByEffort?.low?.effort).toBe('low');+ });});diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/parse-artificial-analysis-effort.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/parse-artificial-analysis-effort.spec.tsnew file mode 100644index 00000000000..32695cc11df--- /dev/null+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/parse-artificial-analysis-effort.spec.ts@@ -0,0 +1,30 @@+import { parseArtificialAnalysisEffort } from '../utils/parse-artificial-analysis-effort.util';++describe('parseArtificialAnalysisEffort', () => {+ it.each([+ ['GPT-5.6 Sol (max)', 'max'],+ ['GPT-5.4 (xhigh)', 'xhigh'],+ ['Grok 4.6 (high)', 'high'],+ ['Gemini 3.5 Flash (medium)', 'medium'],+ ['GPT-5 nano (minimal)', 'minimal'],+ ['Claude Opus 5 (Adaptive Reasoning, Max Effort)', 'max'],+ [+ 'Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)',+ 'max',+ ],+ ['Claude Sonnet 4.6 (Non-reasoning, Low Effort)', 'low'],+ ['GPT-5.6 Luna (Non-reasoning)', 'none'],+ ])('reads "%s" as %s', (displayName, effort) => {+ expect(parseArtificialAnalysisEffort(displayName)).toBe(effort);+ });++ it.each([+ 'Claude Opus 4.5 (Reasoning)',+ 'Gemini 3.1 Pro Preview',+ 'GPT-4o (March 2025, chatgpt-4o-latest)',+ "Llama 3.1 405B (May '24)",+ 'Mistral Large 3 (high) Turbo',+ ])('names no effort for "%s"', (displayName) => {+ expect(parseArtificialAnalysisEffort(displayName)).toBeUndefined();+ });+});diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/read-committed-benchmarks.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/read-committed-benchmarks.spec.tsindex c13c6d60a31..57ef4d2378a 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/read-committed-benchmarks.spec.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/read-committed-benchmarks.spec.ts@@ -65,4 +65,78 @@ describe('readCommittedBenchmarks', () => {expect(result?.benchmark.measuredAt).toBe('2026-09-01');});++ it('recovers per-effort readings under their effort keys, dated when they were taken', () => {+ const index = readCommittedBenchmarks(+ writeOverlay({+ ...PUBLISHED,+ models: {+ 'claude-sonnet-5': {+ ...PUBLISHED.models['claude-sonnet-5'],+ effort: 'max',+ benchmarkByEffort: {+ low: {+ intelligenceIndex: 20.1,+ measuredAt: '2026-09-01',+ aliases: ['Claude Sonnet 5 (low)'],+ },+ },+ },+ },+ }),+ );++ expect(index.get('claudesonnet5')?.effort).toBe('max');+ expect(index.get('claudesonnet5@low')?.intelligenceIndex).toBe(20.1);+ expect(index.get('claudesonnet5@low')?.effort).toBe('low');+ expect(index.get('claudesonnet5@low')?.measuredAt).toBe('2026-09-01');+ expect(index.get('claudesonnet5@max')).toBeUndefined();++ const result = matchBenchmarks({+ modelName: 'claude-sonnet-5',+ siblingModels: {},+ benchmarkIndex: index,+ measuredAt: '2026-09-09',+ efforts: ['low', 'max'],+ });++ expect(result?.benchmarkByEffort?.low?.measuredAt).toBe('2026-09-01');+ expect(result?.benchmarkByEffort?.max).toBeUndefined();+ });++ it('ignores a per-effort key it does not recognise', () => {+ const index = readCommittedBenchmarks(+ writeOverlay({+ ...PUBLISHED,+ models: {+ 'claude-sonnet-5': {+ ...PUBLISHED.models['claude-sonnet-5'],+ benchmarkByEffort: {+ turbo: { intelligenceIndex: 99, aliases: [] },+ },+ },+ },+ }),+ );++ expect([...index.keys()].some((key) => key.includes('@'))).toBe(false);+ });++ it('trusts the key over a nested reading that names another effort', () => {+ const index = readCommittedBenchmarks(+ writeOverlay({+ ...PUBLISHED,+ models: {+ 'claude-sonnet-5': {+ ...PUBLISHED.models['claude-sonnet-5'],+ benchmarkByEffort: {+ low: { intelligenceIndex: 20.1, effort: 'high', aliases: [] },+ },+ },+ },+ }),+ );++ expect(index.get('claudesonnet5@low')?.effort).toBe('low');+ });});diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-match.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-match.type.tsindex 4df7e326474..b15c4734199 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-match.type.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-match.type.ts@@ -1,6 +1,11 @@+import { type AiModelEffort } from 'twenty-shared/ai';+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';+import { type BenchmarkOverlayReading } from './benchmark-overlay-entry.type';+export type BenchmarkMatch = {benchmark: AiModelBenchmark;aliases: string[];+ benchmarkByEffort?: Partial<Record<AiModelEffort, BenchmarkOverlayReading>>;};diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-overlay-entry.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-overlay-entry.type.tsindex baa901b487b..d919958f272 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-overlay-entry.type.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-overlay-entry.type.ts@@ -1,8 +1,16 @@+import { type AiModelEffort } from 'twenty-shared/ai';+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';-// The overlay is the cross-repo artifact: it carries what the catalog embeds-// plus the alias set a joining consumer needs to find the model under the+// A reading plus the alias set a joining consumer needs to find it under the// publisher's own spelling.-export type BenchmarkOverlayEntry = AiModelBenchmark & {+export type BenchmarkOverlayReading = AiModelBenchmark & {aliases: string[];};++// The overlay is the cross-repo artifact: it carries what the catalog embeds+// plus the aliases. Per-effort readings nest under the model's entry so a+// consumer that reads only the top-level fields keeps working.+export type BenchmarkOverlayEntry = BenchmarkOverlayReading & {+ benchmarkByEffort?: Partial<Record<AiModelEffort, BenchmarkOverlayReading>>;+};diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-record.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-record.type.tsindex 2b98fa78508..ae405031bdf 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-record.type.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-record.type.ts@@ -1,7 +1,11 @@+import { type AiModelEffort } from 'twenty-shared/ai';+export type BenchmarkRecord = {intelligenceIndex?: number;outputTokensPerSecond?: number;costPerTask?: number;+ // The effort the publisher's row was measured at, when its name says so.+ effort?: AiModelEffort;// Set only on a record recovered from the committed overlay, so preserved// measurements keep the date they were actually taken.measuredAt?: string;diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/coverage-report.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/coverage-report.type.tsindex ef61d140241..4d595559630 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/types/coverage-report.type.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/coverage-report.type.ts@@ -3,4 +3,7 @@ export type CoverageReport = {scoredGeneralPurposeModelCount: number;unscoredGeneralPurposeModelIds: string[];specializedModelCount: number;+ declaredEffortVariantCount: number;+ scoredEffortVariantCount: number;+ unscoredEffortVariantIds: string[];};diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/generated-model.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/generated-model.type.tsindex 3709d5e351f..1b3311fa6c1 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/types/generated-model.type.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/generated-model.type.ts@@ -22,5 +22,6 @@ export type GeneratedModel = {dataResidency?: DataResidency;zeroDataRetention?: boolean;benchmark?: AiModelBenchmark;+ benchmarkByEffort?: Partial<Record<AiModelEffort, AiModelBenchmark>>;isDeprecated?: boolean;};diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/match-benchmarks-args.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/match-benchmarks-args.type.tsindex 8d6d3bcbd38..6eb1723daa2 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/types/match-benchmarks-args.type.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/match-benchmarks-args.type.ts@@ -1,3 +1,5 @@+import { type AiModelEffort } from 'twenty-shared/ai';+import { type ModelsDevModel } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-model.type';import { type BenchmarkIndex } from './benchmark-index.type';@@ -7,4 +9,5 @@ export type MatchBenchmarksArgs = {siblingModels: Record<string, ModelsDevModel>;benchmarkIndex: BenchmarkIndex;measuredAt: string;+ efforts?: AiModelEffort[];};diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/build-coverage-report.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-coverage-report.util.tsindex 91843c9bfb3..38797d203d8 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/build-coverage-report.util.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-coverage-report.util.ts@@ -18,6 +18,9 @@ export const buildCoverageReport = (scoredGeneralPurposeModelCount: 0,unscoredGeneralPurposeModelIds: [],specializedModelCount: 0,+ declaredEffortVariantCount: 0,+ scoredEffortVariantCount: 0,+ unscoredEffortVariantIds: [],};for (const [providerName, provider] of Object.entries(catalog)) {@@ -40,6 +43,18 @@ export const buildCoverageReport = (`${providerName}/${model.name}`,);}++ for (const effort of model.efforts ?? []) {+ report.declaredEffortVariantCount += 1;++ if (isDefined(model.benchmarkByEffort?.[effort]?.intelligenceIndex)) {+ report.scoredEffortVariantCount += 1;+ } else {+ report.unscoredEffortVariantIds.push(+ `${providerName}/${model.name}@${effort}`,+ );+ }+ }}}diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/build-effort-lookup-key.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-effort-lookup-key.util.tsnew file mode 100644index 00000000000..7a81b08bf10--- /dev/null+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-effort-lookup-key.util.ts@@ -0,0 +1,10 @@+import { type AiModelEffort } from 'twenty-shared/ai';++// A per-effort row is filed beside the bare model key, so the ceiling lookup the+// catalog has always done keeps working while a variant can ask for the figure+// taken at its own effort. `@` survives name normalization, so the two key+// spaces cannot collide.+export const buildEffortLookupKey = (+ normalizedModelName: string,+ effort: AiModelEffort,+): string => `${normalizedModelName}@${effort}`;diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/enrich-catalog.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/enrich-catalog.util.tsindex c172c4011d3..05149da8aea 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/enrich-catalog.util.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/enrich-catalog.util.ts@@ -1,10 +1,27 @@+import { type AiModelEffort } from 'twenty-shared/ai';import { isDefined } from 'twenty-shared/utils';-import { type BenchmarkOverlayEntry } from '../types/benchmark-overlay-entry.type';+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';++import {+ type BenchmarkOverlayEntry,+ type BenchmarkOverlayReading,+} from '../types/benchmark-overlay-entry.type';import { type EnrichCatalogArgs } from '../types/enrich-catalog-args.type';import { matchBenchmarks } from './match-benchmarks.util';+// The catalog embeds the figures only; the aliases stay in the overlay, which+// is where a joining consumer looks for them.+const toCatalogReadings = (+ readings: Partial<Record<AiModelEffort, BenchmarkOverlayReading>>,+): Partial<Record<AiModelEffort, AiModelBenchmark>> =>+ Object.fromEntries(+ Object.entries(readings).map(+ ([effort, { aliases: _aliases, ...reading }]) => [effort, reading],+ ),+ );+export const enrichCatalog = ({catalog,modelsDevData,@@ -22,19 +39,29 @@ export const enrichCatalog = ({siblingModels,benchmarkIndex,measuredAt,+ efforts: model.efforts,});if (!isDefined(match)) {return model;}- overlay[model.name] = { ...match.benchmark, aliases: match.aliases };+ const { benchmark, aliases, benchmarkByEffort } = match;++ overlay[model.name] = {+ ...benchmark,+ aliases,+ ...(isDefined(benchmarkByEffort) ? { benchmarkByEffort } : {}),+ };const { isDeprecated, ...rest } = model;return {...rest,- benchmark: match.benchmark,+ benchmark,+ ...(isDefined(benchmarkByEffort)+ ? { benchmarkByEffort: toCatalogReadings(benchmarkByEffort) }+ : {}),...(isDefined(isDeprecated) ? { isDeprecated } : {}),};});diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.tsindex 25436b5bf77..e6c72a2d33b 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts@@ -1,6 +1,8 @@import { isDefined } from 'twenty-shared/utils';+import { buildEffortLookupKey } from './build-effort-lookup-key.util';import { normalizeModelName } from './normalize-model-name.util';+import { parseArtificialAnalysisEffort } from './parse-artificial-analysis-effort.util';import { type BenchmarkIndex } from '../types/benchmark-index.type';import { type BenchmarkRecord } from '../types/benchmark-record.type';@@ -102,6 +104,14 @@ export const fetchArtificialAnalysisBenchmarks = async (const index: BenchmarkIndex = new Map();+ const file = (key: string, record: BenchmarkRecord): void => {+ const existing = index.get(key);++ if (!isDefined(existing) || preferOver(record, existing)) {+ index.set(key, record);+ }+ };+for (const model of models) {// One malformed row must not cost us every measurement in the response.if (typeof model !== 'object' || model === null) {@@ -127,6 +137,10 @@ export const fetchArtificialAnalysisBenchmarks = async (containerKey: 'cost_per_task',leafKey: 'total_cost',}),+ effort:+ typeof model.name === 'string'+ ? parseArtificialAnalysisEffort(model.name)+ : undefined,aliases,};@@ -136,13 +150,17 @@ export const fetchArtificialAnalysisBenchmarks = async (for (const alias of aliases) {const key = normalizeModelName(alias);- const existing = index.get(key);- if (- key.length > 0 &&- (!isDefined(existing) || preferOver(record, existing))- ) {- index.set(key, record);+ if (key.length === 0) {+ continue;+ }++ // The bare key keeps the ceiling; the effort key lets a pinned variant+ // read the figure taken at its own effort.+ file(key, record);++ if (isDefined(record.effort)) {+ file(buildEffortLookupKey(key, record.effort), record);}}}diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/match-benchmarks.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/match-benchmarks.util.tsindex 983a5a0457b..7dfde6b0fb5 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/match-benchmarks.util.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/match-benchmarks.util.ts@@ -1,19 +1,24 @@+import { type AiModelEffort } from 'twenty-shared/ai';import { isDefined } from 'twenty-shared/utils';+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';+import { type BenchmarkIndex } from '../types/benchmark-index.type';import { type BenchmarkMatch } from '../types/benchmark-match.type';+import { type BenchmarkOverlayReading } from '../types/benchmark-overlay-entry.type';import { type BenchmarkRecord } from '../types/benchmark-record.type';import { type MatchBenchmarksArgs } from '../types/match-benchmarks-args.type';+import { buildEffortLookupKey } from './build-effort-lookup-key.util';import { buildLookupCandidates } from './build-lookup-candidates.util';import { normalizeModelName } from './normalize-model-name.util';const lookup = (index: BenchmarkIndex,- candidates: string[],+ keys: string[],): BenchmarkRecord | undefined => {- for (const candidate of candidates) {- const record = index.get(normalizeModelName(candidate));+ for (const key of keys) {+ const record = index.get(key);if (isDefined(record)) {return record;@@ -23,38 +28,67 @@ const lookup = (return undefined;};+// The publisher lists plenty of models it has measured nothing about, and a row+// of pure aliases says nothing worth carrying into either artifact.+const hasMeasurement = (record: BenchmarkRecord): boolean =>+ [+ record.intelligenceIndex,+ record.outputTokensPerSecond,+ record.costPerTask,+ ].some(isDefined);++const toBenchmark = (+ record: BenchmarkRecord,+ measuredAt: string,+): AiModelBenchmark => ({+ intelligenceIndex: record.intelligenceIndex,+ outputTokensPerSecond: record.outputTokensPerSecond,+ costPerTask: record.costPerTask,+ effort: record.effort,+ measuredAt: record.measuredAt ?? measuredAt,+});+export const matchBenchmarks = ({modelName,siblingModels,benchmarkIndex,measuredAt,+ efforts,}: MatchBenchmarksArgs): BenchmarkMatch | undefined => {- const record = lookup(- benchmarkIndex,- buildLookupCandidates({ modelName, siblingModels }),+ const keys = buildLookupCandidates({ modelName, siblingModels }).map(+ normalizeModelName,);- if (!isDefined(record)) {+ const record = lookup(benchmarkIndex, keys);++ if (!isDefined(record) || !hasMeasurement(record)) {return undefined;}- const { intelligenceIndex, outputTokensPerSecond, costPerTask } = record;+ const benchmarkByEffort: Partial<+ Record<AiModelEffort, BenchmarkOverlayReading>+ > = {};- // The publisher lists plenty of models it has measured nothing about, and a- // row of pure aliases says nothing worth carrying into either artifact.- if (- ![intelligenceIndex, outputTokensPerSecond, costPerTask].some(isDefined)- ) {- return undefined;+ // A declared effort without a row of its own stays blank: the ceiling was+ // measured at another effort and would overstate it.+ for (const effort of efforts ?? []) {+ const effortRecord = lookup(+ benchmarkIndex,+ keys.map((key) => buildEffortLookupKey(key, effort)),+ );++ if (isDefined(effortRecord) && hasMeasurement(effortRecord)) {+ benchmarkByEffort[effort] = {+ ...toBenchmark(effortRecord, measuredAt),+ effort,+ aliases: [...new Set(effortRecord.aliases)],+ };+ }}return {- benchmark: {- intelligenceIndex,- outputTokensPerSecond,- costPerTask,- measuredAt: record.measuredAt ?? measuredAt,- },+ benchmark: toBenchmark(record, measuredAt),aliases: [...new Set([modelName, ...record.aliases])],+ ...(Object.keys(benchmarkByEffort).length > 0 ? { benchmarkByEffort } : {}),};};diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/parse-artificial-analysis-effort.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/parse-artificial-analysis-effort.util.tsnew file mode 100644index 00000000000..40761d3c9f1--- /dev/null+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/parse-artificial-analysis-effort.util.ts@@ -0,0 +1,35 @@+import { type AiModelEffort, isAiModelEffort } from 'twenty-shared/ai';+import { isDefined } from 'twenty-shared/utils';++// A row run with reasoning switched off carries no effort word at all.+const NO_REASONING_MARKERS = new Set(['non-reasoning', 'nonreasoning']);++// The publisher spells the configuration only inside the trailing parentheses+// of a display name: `GPT-5.6 Sol (max)`, `Gemini 3.8 Flash (high)`,+// `Claude Opus 5 (Adaptive Reasoning, Max Effort)`. Dates, fallbacks and a+// bare "Reasoning" in the same place name no effort.+export const parseArtificialAnalysisEffort = (+ displayName: string,+): AiModelEffort | undefined => {+ const configuration = displayName.match(/\(([^()]*)\)\s*$/)?.[1];++ if (!isDefined(configuration)) {+ return undefined;+ }++ const segments = configuration+ .split(',')+ .map((segment) => segment.trim().toLowerCase());++ const effort = segments+ .map((segment) => segment.replace(/\s+effort$/, ''))+ .find(isAiModelEffort);++ if (isDefined(effort)) {+ return effort;+ }++ return segments.some((segment) => NO_REASONING_MARKERS.has(segment))+ ? 'none'+ : undefined;+};diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/read-committed-benchmarks.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/read-committed-benchmarks.util.tsindex e3e40245ed6..19816f33c33 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/read-committed-benchmarks.util.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/read-committed-benchmarks.util.ts@@ -1,14 +1,66 @@import * as fs from 'fs';+import { type AiModelEffort, isAiModelEffort } from 'twenty-shared/ai';++import { buildEffortLookupKey } from './build-effort-lookup-key.util';import { normalizeModelName } from './normalize-model-name.util';import { type BenchmarkIndex } from '../types/benchmark-index.type';import { type BenchmarkRecord } from '../types/benchmark-record.type';-type CommittedEntry = Partial<BenchmarkRecord> & {+type CommittedReading = Partial<BenchmarkRecord> & {measuredAt?: string;aliases?: string[];};+type CommittedEntry = CommittedReading & {+ benchmarkByEffort?: Partial<Record<AiModelEffort, CommittedReading>>;+};++const toRecord = ({+ reading,+ modelName,+ effort,+}: {+ reading: CommittedReading;+ modelName: string;+ effort?: AiModelEffort;+}): BenchmarkRecord => ({+ intelligenceIndex: reading.intelligenceIndex,+ outputTokensPerSecond: reading.outputTokensPerSecond,+ costPerTask: reading.costPerTask,+ // The key a reading is filed under is the authority on its effort.+ effort: effort ?? reading.effort,+ measuredAt: reading.measuredAt,+ aliases: reading.aliases ?? [modelName],+});++const file = ({+ index,+ record,+ modelName,+ toKey,+}: {+ index: BenchmarkIndex;+ record: BenchmarkRecord;+ modelName: string;+ toKey: (normalizedName: string) => string;+}): void => {+ for (const alias of record.aliases) {+ const key = normalizeModelName(alias);++ if (key.length > 0 && !index.has(toKey(key))) {+ index.set(toKey(key), record);+ }+ }++ // An entry always claims its own name, whatever a sibling's alias list says.+ const ownKey = normalizeModelName(modelName);++ if (ownKey.length > 0) {+ index.set(toKey(ownKey), record);+ }+};+// The last published measurements, read back so a failed fetch degrades to// stale data rather than deleting the catalog's benchmarks. Each record keeps// its own measuredAt, so a stale entry stays visibly stale.@@ -24,27 +76,26 @@ export const readCommittedBenchmarks = (filePath: string): BenchmarkIndex => {const index: BenchmarkIndex = new Map();for (const [modelName, entry] of Object.entries(parsed.models ?? {})) {- const record: BenchmarkRecord = {- intelligenceIndex: entry.intelligenceIndex,- outputTokensPerSecond: entry.outputTokensPerSecond,- costPerTask: entry.costPerTask,- measuredAt: entry.measuredAt,- aliases: entry.aliases ?? [modelName],- };-- for (const alias of record.aliases) {- const key = normalizeModelName(alias);-- if (key.length > 0 && !index.has(key)) {- index.set(key, record);- }- }+ file({+ index,+ record: toRecord({ reading: entry, modelName }),+ modelName,+ toKey: (normalizedName) => normalizedName,+ });- // An entry always claims its own name, whatever a sibling's alias list says.- const ownKey = normalizeModelName(modelName);+ for (const [effort, reading] of Object.entries(+ entry.benchmarkByEffort ?? {},+ )) {+ if (!isAiModelEffort(effort)) {+ continue;+ }- if (ownKey.length > 0) {- index.set(ownKey, record);+ file({+ index,+ record: toRecord({ reading, modelName, effort }),+ modelName,+ toKey: (normalizedName) => buildEffortLookupKey(normalizedName, effort),+ });}}diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/render-coverage-report.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/render-coverage-report.util.tsindex 2388caa7c10..0b92a529ebf 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/render-coverage-report.util.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/render-coverage-report.util.ts@@ -9,29 +9,45 @@ const asPercentage = ({}): string =>denominator === 0 ? 'n/a' : `${Math.round((numerator / denominator) * 100)}%`;+const renderGapList = ({+ summary,+ modelIds,+}: {+ summary: string;+ modelIds: string[];+}): string[] =>+ modelIds.length === 0+ ? []+ : [+ '',+ `<details><summary>${summary}</summary>`,+ '',+ ...modelIds.map((modelId) => `- ${modelId}`),+ '',+ '</details>',+ ];+export const renderCoverageReport = (report: CoverageReport): string => {const {generalPurposeModelCount: total,scoredGeneralPurposeModelCount: scored,+ declaredEffortVariantCount: declaredVariants,+ scoredEffortVariantCount: scoredVariants,} = report;- const lines = [+ return ['### Benchmark coverage','',`- General-purpose models with an intelligence index: **${scored}/${total}** (${asPercentage({ numerator: scored, denominator: total })})`,+ `- Declared effort variants with an index measured at their own effort: **${scoredVariants}/${declaredVariants}** (${asPercentage({ numerator: scoredVariants, denominator: declaredVariants })})`,`- Specialized models excluded from the denominator: ${report.specializedModelCount}`,- ];-- if (report.unscoredGeneralPurposeModelIds.length > 0) {- lines.push(- '',- '<details><summary>General-purpose models with no matched intelligence index</summary>',- '',- ...report.unscoredGeneralPurposeModelIds.map((modelId) => `- ${modelId}`),- '',- '</details>',- );- }-- return lines.join('\n');+ ...renderGapList({+ summary: 'General-purpose models with no matched intelligence index',+ modelIds: report.unscoredGeneralPurposeModelIds,+ }),+ ...renderGapList({+ summary: 'Effort variants with no reading at their own effort',+ modelIds: report.unscoredEffortVariantIds,+ }),+ ].join('\n');};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-model-benchmarks-json.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-model-benchmarks-json.spec.tsindex 95539a9c63b..7a0156cc365 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-model-benchmarks-json.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-model-benchmarks-json.spec.ts@@ -1,48 +1,86 @@+import { isAiModelEffort } from 'twenty-shared/ai';+import benchmarkOverlay from 'src/engine/metadata-modules/ai/ai-models/ai-model-benchmarks.json';import defaultAiProviders from 'src/engine/metadata-modules/ai/ai-models/ai-providers.json';import { aiModelBenchmarkSchema } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema';import { type AiProvidersConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-providers-config.type';-const OVERLAY_MODELS = benchmarkOverlay.models as Record<- string,- { aliases: string[] } & Record<string, unknown>->;+type OverlayReading = { aliases: string[] } & Record<string, unknown>;++type OverlayEntry = OverlayReading & {+ benchmarkByEffort?: Record<string, OverlayReading>;+};++const OVERLAY_MODELS = benchmarkOverlay.models as Record<string, OverlayEntry>;const CATALOG_MODELS = Object.values(defaultAiProviders as AiProvidersConfig,).flatMap((provider) => provider.models ?? []);+const SCORED_CATALOG_MODELS = CATALOG_MODELS.filter(+ (model) => model.benchmark !== undefined,+);++const withoutAliases = ({ aliases: _aliases, ...reading }: OverlayReading) =>+ reading;++const toBenchmark = ({+ benchmarkByEffort: _benchmarkByEffort,+ ...reading+}: OverlayEntry) => withoutAliases(reading);+describe('ai-model-benchmarks.json integrity', () => {it('should name the publisher every figure came from', () => {expect(benchmarkOverlay.source).toBe('artificialanalysis.ai');});- it('should pass Zod schema validation for every entry', () => {- Object.values(OVERLAY_MODELS).forEach(({ aliases: _aliases, ...entry }) => {- expect(() => aiModelBenchmarkSchema.parse(entry)).not.toThrow();+ it('should pass Zod schema validation for every entry and per-effort reading', () => {+ Object.values(OVERLAY_MODELS).forEach((entry) => {+ expect(() =>+ aiModelBenchmarkSchema.parse(toBenchmark(entry)),+ ).not.toThrow();++ Object.entries(entry.benchmarkByEffort ?? {}).forEach(+ ([effort, reading]) => {+ expect(isAiModelEffort(effort)).toBe(true);+ expect(() =>+ aiModelBenchmarkSchema.parse(withoutAliases(reading)),+ ).not.toThrow();+ },+ );});});it('should agree with the benchmark merged into the catalog', () => {// The overlay is empty until the sync runs with an API key configured, so// this checks the two artifacts cannot drift rather than pinning a count.- const scored = CATALOG_MODELS.filter(- (model) => model.benchmark !== undefined,- );+ SCORED_CATALOG_MODELS.forEach((model) => {+ const entry = OVERLAY_MODELS[model.name];++ expect(toBenchmark(entry)).toEqual(model.benchmark);+ expect(entry.aliases).toContain(model.name);- scored.forEach((model) => {- const { aliases, ...overlayEntry } = OVERLAY_MODELS[model.name];+ const overlayReadings = Object.fromEntries(+ Object.entries(entry.benchmarkByEffort ?? {}).map(+ ([effort, reading]) => [effort, withoutAliases(reading)],+ ),+ );++ expect(overlayReadings).toEqual(model.benchmarkByEffort ?? {});+ });+ });- expect(overlayEntry).toEqual(model.benchmark);- expect(aliases).toContain(model.name);+ it('should score an effort only where the model declares it', () => {+ CATALOG_MODELS.forEach((model) => {+ Object.keys(model.benchmarkByEffort ?? {}).forEach((effort) => {+ expect(model.efforts ?? []).toContain(effort);+ });});});it('should hold no entry for a model the catalog does not score', () => {const scoredNames = new Set(- CATALOG_MODELS.filter((model) => model.benchmark !== undefined).map(- (model) => model.name,- ),+ SCORED_CATALOG_MODELS.map((model) => model.name),);Object.keys(OVERLAY_MODELS).forEach((name) => {diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-provider-model-config.schema.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-provider-model-config.schema.spec.tsnew file mode 100644index 00000000000..9b9cfbde8bc--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-provider-model-config.schema.spec.ts@@ -0,0 +1,101 @@+import { aiProviderModelConfigSchema } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema';++const MEASURED_AT = '2026-09-10';++const modelWith = (overrides: Record<string, unknown>) => ({+ name: 'gpt-x',+ label: 'GPT X',+ efforts: ['low', 'high'],+ benchmark: { intelligenceIndex: 41, effort: 'high', measuredAt: MEASURED_AT },+ ...overrides,+});++const issuePaths = (model: Record<string, unknown>): string[] => {+ const result = aiProviderModelConfigSchema.safeParse(model);++ return result.success+ ? []+ : result.error.issues.map((issue) => issue.path.join('.'));+};++describe('aiProviderModelConfigSchema', () => {+ it('accepts per-effort readings filed under the effort they were measured at', () => {+ expect(+ issuePaths(+ modelWith({+ benchmarkByEffort: {+ low: {+ intelligenceIndex: 30,+ effort: 'low',+ measuredAt: MEASURED_AT,+ },+ high: { intelligenceIndex: 41, measuredAt: MEASURED_AT },+ },+ }),+ ),+ ).toEqual([]);+ });++ it('rejects a reading filed under another effort than it was measured at', () => {+ // The registry hands a variant the reading under its own key, so this+ // would give the low-effort variant a high-effort figure.+ expect(+ issuePaths(+ modelWith({+ benchmarkByEffort: {+ low: {+ intelligenceIndex: 41,+ effort: 'high',+ measuredAt: MEASURED_AT,+ },+ },+ }),+ ),+ ).toEqual(['benchmarkByEffort.low.effort']);+ });++ it('rejects a reading for an effort the model does not declare', () => {+ expect(+ issuePaths(+ modelWith({+ benchmarkByEffort: {+ max: {+ intelligenceIndex: 45,+ effort: 'max',+ measuredAt: MEASURED_AT,+ },+ },+ }),+ ),+ ).toEqual(['benchmarkByEffort.max']);+ });++ it('rejects per-effort readings on a model that declares no efforts', () => {+ expect(+ issuePaths(+ modelWith({+ efforts: undefined,+ benchmarkByEffort: {+ low: {+ intelligenceIndex: 30,+ effort: 'low',+ measuredAt: MEASURED_AT,+ },+ },+ }),+ ),+ ).toEqual(['benchmarkByEffort.low']);+ });++ it('rejects an effort outside the supported vocabulary', () => {+ expect(+ issuePaths(+ modelWith({+ benchmarkByEffort: {+ turbo: { intelligenceIndex: 30, measuredAt: MEASURED_AT },+ },+ }),+ ),+ ).toContain('benchmarkByEffort.turbo');+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.tsindex fda1a276710..4ad9d74b6f1 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts@@ -211,8 +211,9 @@ export class AiModelRegistryService {modelId: variantId,label: `${baseConfig.label} (${effort})`,effort,- // A benchmark describes the effort it was measured at, not this one.- benchmark: undefined,+ // A reading describes the effort it was taken at, so the variant gets+ // the one taken at its effort or none, never the base model's ceiling.+ benchmark: baseConfig.benchmarkByEffort?.[effort],});const baseModelDef = this.providerModelDefCache.get(baseConfig.modelId);@@ -392,6 +393,7 @@ export class AiModelRegistryService {supportsReasoning: modelDef.supportsReasoning,efforts: modelDef.efforts,benchmark: modelDef.benchmark,+ benchmarkByEffort: modelDef.benchmarkByEffort,isDeprecated: modelDef.isDeprecated,};}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema.tsindex eea17794c84..992320cf69b 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema.ts@@ -1,5 +1,7 @@import { z } from 'zod';+import { AI_MODEL_EFFORTS } from 'twenty-shared/ai';+// One publisher's reading of a model, taken on one date. An index is only// meaningful against models scored the same way, so whoever fills this must use// the same publisher throughout rather than filling gaps from a second. The@@ -8,5 +10,9 @@ export const aiModelBenchmarkSchema = z.object({intelligenceIndex: z.number().optional(),outputTokensPerSecond: z.number().positive().optional(),costPerTask: z.number().positive().optional(),+ // The publisher scores a model once per reasoning effort, so a reading is+ // only comparable with others taken at the same effort. Unset when the+ // publisher's row names none.+ effort: z.enum(AI_MODEL_EFFORTS).optional(),measuredAt: z.string().date(),});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type.tsindex 9d2b592d5e5..92077d86c75 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type.ts@@ -29,5 +29,6 @@ export type AiModelConfig = {// Pinned by a variant id (`model@effort`); unset runs the provider default.effort?: AiModelEffort;benchmark?: AiModelBenchmark;+ benchmarkByEffort?: Partial<Record<AiModelEffort, AiModelBenchmark>>;isDeprecated?: boolean;};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.tsindex 7ef4352efb6..0262add2236 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts@@ -1,6 +1,7 @@import { z } from 'zod';import { AI_MODEL_EFFORTS, DATA_RESIDENCY_KEYS } from 'twenty-shared/ai';+import { isDefined } from 'twenty-shared/utils';import { AI_MODEL_KINDS } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-model-kinds.const';import { aiModelBenchmarkSchema } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema';@@ -33,6 +34,11 @@ export const aiProviderModelConfigSchema = zdataResidency: z.enum(DATA_RESIDENCY_KEYS).optional(),zeroDataRetention: z.boolean().optional(),benchmark: aiModelBenchmarkSchema.optional(),+ // Keyed by effort so a pinned variant carries the reading taken at its own+ // effort instead of the base model's ceiling.+ benchmarkByEffort: z+ .partialRecord(z.enum(AI_MODEL_EFFORTS), aiModelBenchmarkSchema)+ .optional(),isDeprecated: z.boolean().optional(),}).refine(@@ -44,4 +50,28 @@ export const aiProviderModelConfigSchema = zmessage: 'costPerMinute is required for transcription models',path: ['costPerMinute'],},- );+ )+ .superRefine((model, context) => {+ // A variant reads this map by its own effort, so a reading filed under+ // another effort's key, or under an effort no variant can pin, would hand+ // it a figure measured elsewhere.+ for (const [effort, reading] of Object.entries(+ model.benchmarkByEffort ?? {},+ )) {+ if (!(model.efforts ?? []).some((declared) => declared === effort)) {+ context.addIssue({+ code: z.ZodIssueCode.custom,+ message: `benchmarkByEffort.${effort} scores an effort the model does not declare`,+ path: ['benchmarkByEffort', effort],+ });+ }++ if (isDefined(reading?.effort) && reading.effort !== effort) {+ context.addIssue({+ code: z.ZodIssueCode.custom,+ message: `benchmarkByEffort.${effort} carries a reading measured at ${reading.effort}`,+ path: ['benchmarkByEffort', effort, 'effort'],+ });+ }+ }+ });