Skip to content

The open alternative to Salesforce, designed for AI.

Score effort variants from the benchmark rows taken at their own effort (#25730)

Félix Malfait9989ec6bca57
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/enrich-catalog.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/enrich-catalog.spec.ts
index 9a36bf485d1..f9c9c3eb41c 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/enrich-catalog.spec.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/enrich-catalog.spec.ts
@@ -77,4 +77,61 @@ describe('enrichCatalog', () => {
'isDeprecated',
]);
});
+
+ it('embeds per-effort readings in the catalog and mirrors them, with aliases, into the overlay', () => {
+ const catalog = catalogOf('gpt-x');
+
+ catalog.openai.models[0].efforts = ['low', 'high'];
+
+ const overlay = enrich(
+ catalog,
+ new Map([
+ [
+ 'gptx',
+ { intelligenceIndex: 41, effort: 'high', aliases: ['GPT X (high)'] },
+ ],
+ [
+ 'gptx@high',
+ { intelligenceIndex: 41, effort: 'high', aliases: ['GPT X (high)'] },
+ ],
+ [
+ 'gptx@low',
+ { intelligenceIndex: 30, effort: 'low', aliases: ['GPT X (low)'] },
+ ],
+ ]),
+ );
+
+ expect(catalog.openai.models[0].benchmark?.effort).toBe('high');
+ expect(catalog.openai.models[0].benchmarkByEffort).toEqual({
+ low: { intelligenceIndex: 30, effort: 'low', measuredAt: MEASURED_AT },
+ high: { intelligenceIndex: 41, effort: 'high', measuredAt: MEASURED_AT },
+ });
+ expect(overlay['gpt-x'].benchmarkByEffort?.low).toEqual({
+ intelligenceIndex: 30,
+ effort: 'low',
+ measuredAt: MEASURED_AT,
+ aliases: ['GPT X (low)'],
+ });
+ expect(Object.keys(catalog.openai.models[0])).toEqual([
+ 'name',
+ 'label',
+ 'efforts',
+ 'benchmark',
+ 'benchmarkByEffort',
+ ]);
+ });
+
+ it('writes no effort map for a model whose declared efforts have no rows', () => {
+ const catalog = catalogOf('gpt-x');
+
+ catalog.openai.models[0].efforts = ['low', 'high'];
+
+ const overlay = enrich(
+ catalog,
+ new Map([['gptx', { intelligenceIndex: 41, aliases: ['gpt-x'] }]]),
+ );
+
+ expect(catalog.openai.models[0].benchmarkByEffort).toBeUndefined();
+ expect(overlay['gpt-x'].benchmarkByEffort).toBeUndefined();
+ });
});
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts
index e90061e4cfe..49b0aa4855e 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts
@@ -142,4 +142,66 @@ describe('fetchArtificialAnalysisBenchmarks', () => {
expect(record?.intelligenceIndex).toBeUndefined();
expect(record?.outputTokensPerSecond).toBe(92.1);
});
+
+ it('files a row under its effort as well as under the bare model name', async () => {
+ respondWith([
+ {
+ slug: 'gpt-5-6-sol',
+ name: 'GPT-5.6 Sol (max)',
+ evaluations: { artificial_analysis_intelligence_index: 47.1 },
+ },
+ {
+ slug: 'gpt-5-6-sol',
+ name: 'GPT-5.6 Sol (low)',
+ evaluations: { artificial_analysis_intelligence_index: 30.2 },
+ cost_per_task: { total_cost: 0.08 },
+ },
+ ]);
+
+ const index = await fetchArtificialAnalysisBenchmarks('key');
+
+ // The bare key still carries the ceiling, so nothing a consumer reads today
+ // changes.
+ expect(index.get('gpt56sol')?.intelligenceIndex).toBe(47.1);
+ expect(index.get('gpt56sol')?.effort).toBe('max');
+ expect(index.get('gpt56sol@max')?.intelligenceIndex).toBe(47.1);
+ expect(index.get('gpt56sol@low')?.intelligenceIndex).toBe(30.2);
+ expect(index.get('gpt56sol@low')?.costPerTask).toBe(0.08);
+ expect(index.get('gpt56sol@low')?.effort).toBe('low');
+ });
+
+ it('keeps the better-measured row when two rows share an effort', async () => {
+ respondWith([
+ {
+ slug: 'grok-4-6',
+ name: 'Grok 4.6 (high)',
+ evaluations: { artificial_analysis_intelligence_index: 44.4 },
+ },
+ {
+ slug: 'grok-4-6',
+ name: 'Grok 4.6 (high)',
+ evaluations: { artificial_analysis_intelligence_index: 44.4 },
+ median_output_tokens_per_second: 71,
+ },
+ ]);
+
+ const index = await fetchArtificialAnalysisBenchmarks('key');
+
+ expect(index.get('grok46@high')?.outputTokensPerSecond).toBe(71);
+ });
+
+ it('files a row naming no effort under the bare name only', async () => {
+ respondWith([
+ {
+ slug: 'gemini-3-1-pro-preview',
+ name: 'Gemini 3.1 Pro Preview',
+ evaluations: { artificial_analysis_intelligence_index: 30.4 },
+ },
+ ]);
+
+ const index = await fetchArtificialAnalysisBenchmarks('key');
+
+ expect(index.get('gemini31propreview')?.effort).toBeUndefined();
+ expect([...index.keys()].some((key) => key.includes('@'))).toBe(false);
+ });
});
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts
index 5afeb2f3527..39c07ea94a9 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts
@@ -217,4 +217,123 @@ describe('matchBenchmarks', () => {
'mistralai/mistral-large-2512',
]);
});
+
+ it('reads each declared effort from the row taken at that effort', () => {
+ const result = matchBenchmarks({
+ modelName: 'mistral-large-2512',
+ siblingModels: MISTRAL_MODELS,
+ benchmarkIndex: indexOf({
+ mistrallarge2512: {
+ intelligenceIndex: 41,
+ effort: 'high',
+ aliases: ['Mistral Large 3 (high)'],
+ },
+ 'mistrallarge2512@high': {
+ intelligenceIndex: 41,
+ effort: 'high',
+ aliases: ['Mistral Large 3 (high)'],
+ },
+ 'mistrallarge2512@none': {
+ intelligenceIndex: 22,
+ costPerTask: 0.1,
+ effort: 'none',
+ aliases: ['Mistral Large 3 (Non-reasoning)'],
+ },
+ }),
+ measuredAt: MEASURED_AT,
+ efforts: ['none', 'high'],
+ });
+
+ expect(result?.benchmark.effort).toBe('high');
+ expect(result?.benchmarkByEffort?.none).toEqual({
+ intelligenceIndex: 22,
+ costPerTask: 0.1,
+ effort: 'none',
+ measuredAt: MEASURED_AT,
+ aliases: ['Mistral Large 3 (Non-reasoning)'],
+ });
+ expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);
+ });
+
+ it('leaves a declared effort without its own row blank rather than lending it the ceiling', () => {
+ const result = matchBenchmarks({
+ modelName: 'mistral-large-2512',
+ siblingModels: MISTRAL_MODELS,
+ benchmarkIndex: indexOf({
+ mistrallarge2512: {
+ intelligenceIndex: 41,
+ effort: 'high',
+ aliases: [],
+ },
+ 'mistrallarge2512@high': {
+ intelligenceIndex: 41,
+ effort: 'high',
+ aliases: [],
+ },
+ }),
+ measuredAt: MEASURED_AT,
+ efforts: ['none', 'high'],
+ });
+
+ expect(result?.benchmarkByEffort?.none).toBeUndefined();
+ expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);
+ });
+
+ it('carries no effort map at all when no declared effort has a row', () => {
+ const result = matchBenchmarks({
+ modelName: 'mistral-large-2512',
+ siblingModels: MISTRAL_MODELS,
+ benchmarkIndex: indexOf({
+ mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+ }),
+ measuredAt: MEASURED_AT,
+ efforts: ['none', 'high'],
+ });
+
+ expect(result?.benchmark.intelligenceIndex).toBe(41);
+ expect(result?.benchmarkByEffort).toBeUndefined();
+ });
+
+ it('resolves a rolling alias to its release for per-effort rows too', () => {
+ const result = matchBenchmarks({
+ modelName: 'mistral-large-latest',
+ siblingModels: MISTRAL_MODELS,
+ benchmarkIndex: indexOf({
+ mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+ 'mistrallarge2512@high': {
+ intelligenceIndex: 41,
+ effort: 'high',
+ aliases: [],
+ },
+ 'mistrallarge@high': {
+ intelligenceIndex: 20,
+ effort: 'high',
+ aliases: [],
+ },
+ }),
+ measuredAt: MEASURED_AT,
+ efforts: ['high'],
+ });
+
+ expect(result?.benchmarkByEffort?.high?.intelligenceIndex).toBe(41);
+ });
+
+ it('labels a per-effort reading with the key it was filed under', () => {
+ const result = matchBenchmarks({
+ modelName: 'mistral-large-2512',
+ siblingModels: MISTRAL_MODELS,
+ benchmarkIndex: indexOf({
+ mistrallarge2512: { intelligenceIndex: 41, aliases: [] },
+ 'mistrallarge2512@low': {
+ intelligenceIndex: 30,
+ effort: 'high',
+ aliases: [],
+ },
+ }),
+ measuredAt: MEASURED_AT,
+ efforts: ['low'],
+ });
+
+ expect(result?.benchmarkByEffort?.low?.effort).toBe('low');
+ });
});
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/parse-artificial-analysis-effort.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/parse-artificial-analysis-effort.spec.ts
new file mode 100644
index 00000000000..32695cc11df
--- /dev/null
+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/parse-artificial-analysis-effort.spec.ts
@@ -0,0 +1,30 @@
+import { parseArtificialAnalysisEffort } from '../utils/parse-artificial-analysis-effort.util';
+
+describe('parseArtificialAnalysisEffort', () => {
+ it.each([
+ ['GPT-5.6 Sol (max)', 'max'],
+ ['GPT-5.4 (xhigh)', 'xhigh'],
+ ['Grok 4.6 (high)', 'high'],
+ ['Gemini 3.5 Flash (medium)', 'medium'],
+ ['GPT-5 nano (minimal)', 'minimal'],
+ ['Claude Opus 5 (Adaptive Reasoning, Max Effort)', 'max'],
+ [
+ 'Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)',
+ 'max',
+ ],
+ ['Claude Sonnet 4.6 (Non-reasoning, Low Effort)', 'low'],
+ ['GPT-5.6 Luna (Non-reasoning)', 'none'],
+ ])('reads "%s" as %s', (displayName, effort) => {
+ expect(parseArtificialAnalysisEffort(displayName)).toBe(effort);
+ });
+
+ it.each([
+ 'Claude Opus 4.5 (Reasoning)',
+ 'Gemini 3.1 Pro Preview',
+ 'GPT-4o (March 2025, chatgpt-4o-latest)',
+ "Llama 3.1 405B (May '24)",
+ 'Mistral Large 3 (high) Turbo',
+ ])('names no effort for "%s"', (displayName) => {
+ expect(parseArtificialAnalysisEffort(displayName)).toBeUndefined();
+ });
+});
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/read-committed-benchmarks.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/read-committed-benchmarks.spec.ts
index c13c6d60a31..57ef4d2378a 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/read-committed-benchmarks.spec.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/read-committed-benchmarks.spec.ts
@@ -65,4 +65,78 @@ describe('readCommittedBenchmarks', () => {
expect(result?.benchmark.measuredAt).toBe('2026-09-01');
});
+
+ it('recovers per-effort readings under their effort keys, dated when they were taken', () => {
+ const index = readCommittedBenchmarks(
+ writeOverlay({
+ ...PUBLISHED,
+ models: {
+ 'claude-sonnet-5': {
+ ...PUBLISHED.models['claude-sonnet-5'],
+ effort: 'max',
+ benchmarkByEffort: {
+ low: {
+ intelligenceIndex: 20.1,
+ measuredAt: '2026-09-01',
+ aliases: ['Claude Sonnet 5 (low)'],
+ },
+ },
+ },
+ },
+ }),
+ );
+
+ expect(index.get('claudesonnet5')?.effort).toBe('max');
+ expect(index.get('claudesonnet5@low')?.intelligenceIndex).toBe(20.1);
+ expect(index.get('claudesonnet5@low')?.effort).toBe('low');
+ expect(index.get('claudesonnet5@low')?.measuredAt).toBe('2026-09-01');
+ expect(index.get('claudesonnet5@max')).toBeUndefined();
+
+ const result = matchBenchmarks({
+ modelName: 'claude-sonnet-5',
+ siblingModels: {},
+ benchmarkIndex: index,
+ measuredAt: '2026-09-09',
+ efforts: ['low', 'max'],
+ });
+
+ expect(result?.benchmarkByEffort?.low?.measuredAt).toBe('2026-09-01');
+ expect(result?.benchmarkByEffort?.max).toBeUndefined();
+ });
+
+ it('ignores a per-effort key it does not recognise', () => {
+ const index = readCommittedBenchmarks(
+ writeOverlay({
+ ...PUBLISHED,
+ models: {
+ 'claude-sonnet-5': {
+ ...PUBLISHED.models['claude-sonnet-5'],
+ benchmarkByEffort: {
+ turbo: { intelligenceIndex: 99, aliases: [] },
+ },
+ },
+ },
+ }),
+ );
+
+ expect([...index.keys()].some((key) => key.includes('@'))).toBe(false);
+ });
+
+ it('trusts the key over a nested reading that names another effort', () => {
+ const index = readCommittedBenchmarks(
+ writeOverlay({
+ ...PUBLISHED,
+ models: {
+ 'claude-sonnet-5': {
+ ...PUBLISHED.models['claude-sonnet-5'],
+ benchmarkByEffort: {
+ low: { intelligenceIndex: 20.1, effort: 'high', aliases: [] },
+ },
+ },
+ },
+ }),
+ );
+
+ expect(index.get('claudesonnet5@low')?.effort).toBe('low');
+ });
});
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-match.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-match.type.ts
index 4df7e326474..b15c4734199 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-match.type.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-match.type.ts
@@ -1,6 +1,11 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
+import { type BenchmarkOverlayReading } from './benchmark-overlay-entry.type';
+
export type BenchmarkMatch = {
benchmark: AiModelBenchmark;
aliases: string[];
+ benchmarkByEffort?: Partial<Record<AiModelEffort, BenchmarkOverlayReading>>;
};
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-overlay-entry.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-overlay-entry.type.ts
index baa901b487b..d919958f272 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-overlay-entry.type.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-overlay-entry.type.ts
@@ -1,8 +1,16 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
-// The overlay is the cross-repo artifact: it carries what the catalog embeds
-// plus the alias set a joining consumer needs to find the model under the
+// A reading plus the alias set a joining consumer needs to find it under the
// publisher's own spelling.
-export type BenchmarkOverlayEntry = AiModelBenchmark & {
+export type BenchmarkOverlayReading = AiModelBenchmark & {
aliases: string[];
};
+
+// The overlay is the cross-repo artifact: it carries what the catalog embeds
+// plus the aliases. Per-effort readings nest under the model's entry so a
+// consumer that reads only the top-level fields keeps working.
+export type BenchmarkOverlayEntry = BenchmarkOverlayReading & {
+ benchmarkByEffort?: Partial<Record<AiModelEffort, BenchmarkOverlayReading>>;
+};
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-record.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-record.type.ts
index 2b98fa78508..ae405031bdf 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-record.type.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/benchmark-record.type.ts
@@ -1,7 +1,11 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
export type BenchmarkRecord = {
intelligenceIndex?: number;
outputTokensPerSecond?: number;
costPerTask?: number;
+ // The effort the publisher's row was measured at, when its name says so.
+ effort?: AiModelEffort;
// Set only on a record recovered from the committed overlay, so preserved
// measurements keep the date they were actually taken.
measuredAt?: string;
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/coverage-report.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/coverage-report.type.ts
index ef61d140241..4d595559630 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/types/coverage-report.type.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/coverage-report.type.ts
@@ -3,4 +3,7 @@ export type CoverageReport = {
scoredGeneralPurposeModelCount: number;
unscoredGeneralPurposeModelIds: string[];
specializedModelCount: number;
+ declaredEffortVariantCount: number;
+ scoredEffortVariantCount: number;
+ unscoredEffortVariantIds: string[];
};
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/generated-model.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/generated-model.type.ts
index 3709d5e351f..1b3311fa6c1 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/types/generated-model.type.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/generated-model.type.ts
@@ -22,5 +22,6 @@ export type GeneratedModel = {
dataResidency?: DataResidency;
zeroDataRetention?: boolean;
benchmark?: AiModelBenchmark;
+ benchmarkByEffort?: Partial<Record<AiModelEffort, AiModelBenchmark>>;
isDeprecated?: boolean;
};
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/types/match-benchmarks-args.type.ts b/packages/twenty-server/scripts/ai-catalog-sync/types/match-benchmarks-args.type.ts
index 8d6d3bcbd38..6eb1723daa2 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/types/match-benchmarks-args.type.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/types/match-benchmarks-args.type.ts
@@ -1,3 +1,5 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
import { type ModelsDevModel } from 'src/engine/metadata-modules/ai/ai-models/types/models-dev-model.type';
import { type BenchmarkIndex } from './benchmark-index.type';
@@ -7,4 +9,5 @@ export type MatchBenchmarksArgs = {
siblingModels: Record<string, ModelsDevModel>;
benchmarkIndex: BenchmarkIndex;
measuredAt: string;
+ efforts?: AiModelEffort[];
};
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/build-coverage-report.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-coverage-report.util.ts
index 91843c9bfb3..38797d203d8 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/build-coverage-report.util.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-coverage-report.util.ts
@@ -18,6 +18,9 @@ export const buildCoverageReport = (
scoredGeneralPurposeModelCount: 0,
unscoredGeneralPurposeModelIds: [],
specializedModelCount: 0,
+ declaredEffortVariantCount: 0,
+ scoredEffortVariantCount: 0,
+ unscoredEffortVariantIds: [],
};
for (const [providerName, provider] of Object.entries(catalog)) {
@@ -40,6 +43,18 @@ export const buildCoverageReport = (
`${providerName}/${model.name}`,
);
}
+
+ for (const effort of model.efforts ?? []) {
+ report.declaredEffortVariantCount += 1;
+
+ if (isDefined(model.benchmarkByEffort?.[effort]?.intelligenceIndex)) {
+ report.scoredEffortVariantCount += 1;
+ } else {
+ report.unscoredEffortVariantIds.push(
+ `${providerName}/${model.name}@${effort}`,
+ );
+ }
+ }
}
}
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/build-effort-lookup-key.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-effort-lookup-key.util.ts
new file mode 100644
index 00000000000..7a81b08bf10
--- /dev/null
+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-effort-lookup-key.util.ts
@@ -0,0 +1,10 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
+
+// A per-effort row is filed beside the bare model key, so the ceiling lookup the
+// catalog has always done keeps working while a variant can ask for the figure
+// taken at its own effort. `@` survives name normalization, so the two key
+// spaces cannot collide.
+export const buildEffortLookupKey = (
+ normalizedModelName: string,
+ effort: AiModelEffort,
+): string => `${normalizedModelName}@${effort}`;
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/enrich-catalog.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/enrich-catalog.util.ts
index c172c4011d3..05149da8aea 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/enrich-catalog.util.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/enrich-catalog.util.ts
@@ -1,10 +1,27 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
import { isDefined } from 'twenty-shared/utils';
-import { type BenchmarkOverlayEntry } from '../types/benchmark-overlay-entry.type';
+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
+
+import {
+ type BenchmarkOverlayEntry,
+ type BenchmarkOverlayReading,
+} from '../types/benchmark-overlay-entry.type';
import { type EnrichCatalogArgs } from '../types/enrich-catalog-args.type';
import { matchBenchmarks } from './match-benchmarks.util';
+// The catalog embeds the figures only; the aliases stay in the overlay, which
+// is where a joining consumer looks for them.
+const toCatalogReadings = (
+ readings: Partial<Record<AiModelEffort, BenchmarkOverlayReading>>,
+): Partial<Record<AiModelEffort, AiModelBenchmark>> =>
+ Object.fromEntries(
+ Object.entries(readings).map(
+ ([effort, { aliases: _aliases, ...reading }]) => [effort, reading],
+ ),
+ );
+
export const enrichCatalog = ({
catalog,
modelsDevData,
@@ -22,19 +39,29 @@ export const enrichCatalog = ({
siblingModels,
benchmarkIndex,
measuredAt,
+ efforts: model.efforts,
});
if (!isDefined(match)) {
return model;
}
- overlay[model.name] = { ...match.benchmark, aliases: match.aliases };
+ const { benchmark, aliases, benchmarkByEffort } = match;
+
+ overlay[model.name] = {
+ ...benchmark,
+ aliases,
+ ...(isDefined(benchmarkByEffort) ? { benchmarkByEffort } : {}),
+ };
const { isDeprecated, ...rest } = model;
return {
...rest,
- benchmark: match.benchmark,
+ benchmark,
+ ...(isDefined(benchmarkByEffort)
+ ? { benchmarkByEffort: toCatalogReadings(benchmarkByEffort) }
+ : {}),
...(isDefined(isDeprecated) ? { isDeprecated } : {}),
};
});
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts
index 25436b5bf77..e6c72a2d33b 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts
@@ -1,6 +1,8 @@
import { isDefined } from 'twenty-shared/utils';
+import { buildEffortLookupKey } from './build-effort-lookup-key.util';
import { normalizeModelName } from './normalize-model-name.util';
+import { parseArtificialAnalysisEffort } from './parse-artificial-analysis-effort.util';
import { type BenchmarkIndex } from '../types/benchmark-index.type';
import { type BenchmarkRecord } from '../types/benchmark-record.type';
@@ -102,6 +104,14 @@ export const fetchArtificialAnalysisBenchmarks = async (
const index: BenchmarkIndex = new Map();
+ const file = (key: string, record: BenchmarkRecord): void => {
+ const existing = index.get(key);
+
+ if (!isDefined(existing) || preferOver(record, existing)) {
+ index.set(key, record);
+ }
+ };
+
for (const model of models) {
// One malformed row must not cost us every measurement in the response.
if (typeof model !== 'object' || model === null) {
@@ -127,6 +137,10 @@ export const fetchArtificialAnalysisBenchmarks = async (
containerKey: 'cost_per_task',
leafKey: 'total_cost',
}),
+ effort:
+ typeof model.name === 'string'
+ ? parseArtificialAnalysisEffort(model.name)
+ : undefined,
aliases,
};
@@ -136,13 +150,17 @@ export const fetchArtificialAnalysisBenchmarks = async (
for (const alias of aliases) {
const key = normalizeModelName(alias);
- const existing = index.get(key);
- if (
- key.length > 0 &&
- (!isDefined(existing) || preferOver(record, existing))
- ) {
- index.set(key, record);
+ if (key.length === 0) {
+ continue;
+ }
+
+ // The bare key keeps the ceiling; the effort key lets a pinned variant
+ // read the figure taken at its own effort.
+ file(key, record);
+
+ if (isDefined(record.effort)) {
+ file(buildEffortLookupKey(key, record.effort), record);
}
}
}
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/match-benchmarks.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/match-benchmarks.util.ts
index 983a5a0457b..7dfde6b0fb5 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/match-benchmarks.util.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/match-benchmarks.util.ts
@@ -1,19 +1,24 @@
+import { type AiModelEffort } from 'twenty-shared/ai';
import { isDefined } from 'twenty-shared/utils';
+import { type AiModelBenchmark } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.type';
+
import { type BenchmarkIndex } from '../types/benchmark-index.type';
import { type BenchmarkMatch } from '../types/benchmark-match.type';
+import { type BenchmarkOverlayReading } from '../types/benchmark-overlay-entry.type';
import { type BenchmarkRecord } from '../types/benchmark-record.type';
import { type MatchBenchmarksArgs } from '../types/match-benchmarks-args.type';
+import { buildEffortLookupKey } from './build-effort-lookup-key.util';
import { buildLookupCandidates } from './build-lookup-candidates.util';
import { normalizeModelName } from './normalize-model-name.util';
const lookup = (
index: BenchmarkIndex,
- candidates: string[],
+ keys: string[],
): BenchmarkRecord | undefined => {
- for (const candidate of candidates) {
- const record = index.get(normalizeModelName(candidate));
+ for (const key of keys) {
+ const record = index.get(key);
if (isDefined(record)) {
return record;
@@ -23,38 +28,67 @@ const lookup = (
return undefined;
};
+// The publisher lists plenty of models it has measured nothing about, and a row
+// of pure aliases says nothing worth carrying into either artifact.
+const hasMeasurement = (record: BenchmarkRecord): boolean =>
+ [
+ record.intelligenceIndex,
+ record.outputTokensPerSecond,
+ record.costPerTask,
+ ].some(isDefined);
+
+const toBenchmark = (
+ record: BenchmarkRecord,
+ measuredAt: string,
+): AiModelBenchmark => ({
+ intelligenceIndex: record.intelligenceIndex,
+ outputTokensPerSecond: record.outputTokensPerSecond,
+ costPerTask: record.costPerTask,
+ effort: record.effort,
+ measuredAt: record.measuredAt ?? measuredAt,
+});
+
export const matchBenchmarks = ({
modelName,
siblingModels,
benchmarkIndex,
measuredAt,
+ efforts,
}: MatchBenchmarksArgs): BenchmarkMatch | undefined => {
- const record = lookup(
- benchmarkIndex,
- buildLookupCandidates({ modelName, siblingModels }),
+ const keys = buildLookupCandidates({ modelName, siblingModels }).map(
+ normalizeModelName,
);
- if (!isDefined(record)) {
+ const record = lookup(benchmarkIndex, keys);
+
+ if (!isDefined(record) || !hasMeasurement(record)) {
return undefined;
}
- const { intelligenceIndex, outputTokensPerSecond, costPerTask } = record;
+ const benchmarkByEffort: Partial<
+ Record<AiModelEffort, BenchmarkOverlayReading>
+ > = {};
- // The publisher lists plenty of models it has measured nothing about, and a
- // row of pure aliases says nothing worth carrying into either artifact.
- if (
- ![intelligenceIndex, outputTokensPerSecond, costPerTask].some(isDefined)
- ) {
- return undefined;
+ // A declared effort without a row of its own stays blank: the ceiling was
+ // measured at another effort and would overstate it.
+ for (const effort of efforts ?? []) {
+ const effortRecord = lookup(
+ benchmarkIndex,
+ keys.map((key) => buildEffortLookupKey(key, effort)),
+ );
+
+ if (isDefined(effortRecord) && hasMeasurement(effortRecord)) {
+ benchmarkByEffort[effort] = {
+ ...toBenchmark(effortRecord, measuredAt),
+ effort,
+ aliases: [...new Set(effortRecord.aliases)],
+ };
+ }
}
return {
- benchmark: {
- intelligenceIndex,
- outputTokensPerSecond,
- costPerTask,
- measuredAt: record.measuredAt ?? measuredAt,
- },
+ benchmark: toBenchmark(record, measuredAt),
aliases: [...new Set([modelName, ...record.aliases])],
+ ...(Object.keys(benchmarkByEffort).length > 0 ? { benchmarkByEffort } : {}),
};
};
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/parse-artificial-analysis-effort.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/parse-artificial-analysis-effort.util.ts
new file mode 100644
index 00000000000..40761d3c9f1
--- /dev/null
+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/parse-artificial-analysis-effort.util.ts
@@ -0,0 +1,35 @@
+import { type AiModelEffort, isAiModelEffort } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
+
+// A row run with reasoning switched off carries no effort word at all.
+const NO_REASONING_MARKERS = new Set(['non-reasoning', 'nonreasoning']);
+
+// The publisher spells the configuration only inside the trailing parentheses
+// of a display name: `GPT-5.6 Sol (max)`, `Gemini 3.8 Flash (high)`,
+// `Claude Opus 5 (Adaptive Reasoning, Max Effort)`. Dates, fallbacks and a
+// bare "Reasoning" in the same place name no effort.
+export const parseArtificialAnalysisEffort = (
+ displayName: string,
+): AiModelEffort | undefined => {
+ const configuration = displayName.match(/\(([^()]*)\)\s*$/)?.[1];
+
+ if (!isDefined(configuration)) {
+ return undefined;
+ }
+
+ const segments = configuration
+ .split(',')
+ .map((segment) => segment.trim().toLowerCase());
+
+ const effort = segments
+ .map((segment) => segment.replace(/\s+effort$/, ''))
+ .find(isAiModelEffort);
+
+ if (isDefined(effort)) {
+ return effort;
+ }
+
+ return segments.some((segment) => NO_REASONING_MARKERS.has(segment))
+ ? 'none'
+ : undefined;
+};
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/read-committed-benchmarks.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/read-committed-benchmarks.util.ts
index e3e40245ed6..19816f33c33 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/read-committed-benchmarks.util.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/read-committed-benchmarks.util.ts
@@ -1,14 +1,66 @@
import * as fs from 'fs';
+import { type AiModelEffort, isAiModelEffort } from 'twenty-shared/ai';
+
+import { buildEffortLookupKey } from './build-effort-lookup-key.util';
import { normalizeModelName } from './normalize-model-name.util';
import { type BenchmarkIndex } from '../types/benchmark-index.type';
import { type BenchmarkRecord } from '../types/benchmark-record.type';
-type CommittedEntry = Partial<BenchmarkRecord> & {
+type CommittedReading = Partial<BenchmarkRecord> & {
measuredAt?: string;
aliases?: string[];
};
+type CommittedEntry = CommittedReading & {
+ benchmarkByEffort?: Partial<Record<AiModelEffort, CommittedReading>>;
+};
+
+const toRecord = ({
+ reading,
+ modelName,
+ effort,
+}: {
+ reading: CommittedReading;
+ modelName: string;
+ effort?: AiModelEffort;
+}): BenchmarkRecord => ({
+ intelligenceIndex: reading.intelligenceIndex,
+ outputTokensPerSecond: reading.outputTokensPerSecond,
+ costPerTask: reading.costPerTask,
+ // The key a reading is filed under is the authority on its effort.
+ effort: effort ?? reading.effort,
+ measuredAt: reading.measuredAt,
+ aliases: reading.aliases ?? [modelName],
+});
+
+const file = ({
+ index,
+ record,
+ modelName,
+ toKey,
+}: {
+ index: BenchmarkIndex;
+ record: BenchmarkRecord;
+ modelName: string;
+ toKey: (normalizedName: string) => string;
+}): void => {
+ for (const alias of record.aliases) {
+ const key = normalizeModelName(alias);
+
+ if (key.length > 0 && !index.has(toKey(key))) {
+ index.set(toKey(key), record);
+ }
+ }
+
+ // An entry always claims its own name, whatever a sibling's alias list says.
+ const ownKey = normalizeModelName(modelName);
+
+ if (ownKey.length > 0) {
+ index.set(toKey(ownKey), record);
+ }
+};
+
// The last published measurements, read back so a failed fetch degrades to
// stale data rather than deleting the catalog's benchmarks. Each record keeps
// its own measuredAt, so a stale entry stays visibly stale.
@@ -24,27 +76,26 @@ export const readCommittedBenchmarks = (filePath: string): BenchmarkIndex => {
const index: BenchmarkIndex = new Map();
for (const [modelName, entry] of Object.entries(parsed.models ?? {})) {
- const record: BenchmarkRecord = {
- intelligenceIndex: entry.intelligenceIndex,
- outputTokensPerSecond: entry.outputTokensPerSecond,
- costPerTask: entry.costPerTask,
- measuredAt: entry.measuredAt,
- aliases: entry.aliases ?? [modelName],
- };
-
- for (const alias of record.aliases) {
- const key = normalizeModelName(alias);
-
- if (key.length > 0 && !index.has(key)) {
- index.set(key, record);
- }
- }
+ file({
+ index,
+ record: toRecord({ reading: entry, modelName }),
+ modelName,
+ toKey: (normalizedName) => normalizedName,
+ });
- // An entry always claims its own name, whatever a sibling's alias list says.
- const ownKey = normalizeModelName(modelName);
+ for (const [effort, reading] of Object.entries(
+ entry.benchmarkByEffort ?? {},
+ )) {
+ if (!isAiModelEffort(effort)) {
+ continue;
+ }
- if (ownKey.length > 0) {
- index.set(ownKey, record);
+ file({
+ index,
+ record: toRecord({ reading, modelName, effort }),
+ modelName,
+ toKey: (normalizedName) => buildEffortLookupKey(normalizedName, effort),
+ });
}
}
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/render-coverage-report.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/render-coverage-report.util.ts
index 2388caa7c10..0b92a529ebf 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/render-coverage-report.util.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/render-coverage-report.util.ts
@@ -9,29 +9,45 @@ const asPercentage = ({
}): string =>
denominator === 0 ? 'n/a' : `${Math.round((numerator / denominator) * 100)}%`;
+const renderGapList = ({
+ summary,
+ modelIds,
+}: {
+ summary: string;
+ modelIds: string[];
+}): string[] =>
+ modelIds.length === 0
+ ? []
+ : [
+ '',
+ `<details><summary>${summary}</summary>`,
+ '',
+ ...modelIds.map((modelId) => `- ${modelId}`),
+ '',
+ '</details>',
+ ];
+
export const renderCoverageReport = (report: CoverageReport): string => {
const {
generalPurposeModelCount: total,
scoredGeneralPurposeModelCount: scored,
+ declaredEffortVariantCount: declaredVariants,
+ scoredEffortVariantCount: scoredVariants,
} = report;
- const lines = [
+ return [
'### Benchmark coverage',
'',
`- General-purpose models with an intelligence index: **${scored}/${total}** (${asPercentage({ numerator: scored, denominator: total })})`,
+ `- Declared effort variants with an index measured at their own effort: **${scoredVariants}/${declaredVariants}** (${asPercentage({ numerator: scoredVariants, denominator: declaredVariants })})`,
`- Specialized models excluded from the denominator: ${report.specializedModelCount}`,
- ];
-
- if (report.unscoredGeneralPurposeModelIds.length > 0) {
- lines.push(
- '',
- '<details><summary>General-purpose models with no matched intelligence index</summary>',
- '',
- ...report.unscoredGeneralPurposeModelIds.map((modelId) => `- ${modelId}`),
- '',
- '</details>',
- );
- }
-
- return lines.join('\n');
+ ...renderGapList({
+ summary: 'General-purpose models with no matched intelligence index',
+ modelIds: report.unscoredGeneralPurposeModelIds,
+ }),
+ ...renderGapList({
+ summary: 'Effort variants with no reading at their own effort',
+ modelIds: report.unscoredEffortVariantIds,
+ }),
+ ].join('\n');
};
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-model-benchmarks-json.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-model-benchmarks-json.spec.ts
index 95539a9c63b..7a0156cc365 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-model-benchmarks-json.spec.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-model-benchmarks-json.spec.ts
@@ -1,48 +1,86 @@
+import { isAiModelEffort } from 'twenty-shared/ai';
+
import benchmarkOverlay from 'src/engine/metadata-modules/ai/ai-models/ai-model-benchmarks.json';
import defaultAiProviders from 'src/engine/metadata-modules/ai/ai-models/ai-providers.json';
import { aiModelBenchmarkSchema } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema';
import { type AiProvidersConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-providers-config.type';
-const OVERLAY_MODELS = benchmarkOverlay.models as Record<
- string,
- { aliases: string[] } & Record<string, unknown>
->;
+type OverlayReading = { aliases: string[] } & Record<string, unknown>;
+
+type OverlayEntry = OverlayReading & {
+ benchmarkByEffort?: Record<string, OverlayReading>;
+};
+
+const OVERLAY_MODELS = benchmarkOverlay.models as Record<string, OverlayEntry>;
const CATALOG_MODELS = Object.values(
defaultAiProviders as AiProvidersConfig,
).flatMap((provider) => provider.models ?? []);
+const SCORED_CATALOG_MODELS = CATALOG_MODELS.filter(
+ (model) => model.benchmark !== undefined,
+);
+
+const withoutAliases = ({ aliases: _aliases, ...reading }: OverlayReading) =>
+ reading;
+
+const toBenchmark = ({
+ benchmarkByEffort: _benchmarkByEffort,
+ ...reading
+}: OverlayEntry) => withoutAliases(reading);
+
describe('ai-model-benchmarks.json integrity', () => {
it('should name the publisher every figure came from', () => {
expect(benchmarkOverlay.source).toBe('artificialanalysis.ai');
});
- it('should pass Zod schema validation for every entry', () => {
- Object.values(OVERLAY_MODELS).forEach(({ aliases: _aliases, ...entry }) => {
- expect(() => aiModelBenchmarkSchema.parse(entry)).not.toThrow();
+ it('should pass Zod schema validation for every entry and per-effort reading', () => {
+ Object.values(OVERLAY_MODELS).forEach((entry) => {
+ expect(() =>
+ aiModelBenchmarkSchema.parse(toBenchmark(entry)),
+ ).not.toThrow();
+
+ Object.entries(entry.benchmarkByEffort ?? {}).forEach(
+ ([effort, reading]) => {
+ expect(isAiModelEffort(effort)).toBe(true);
+ expect(() =>
+ aiModelBenchmarkSchema.parse(withoutAliases(reading)),
+ ).not.toThrow();
+ },
+ );
});
});
it('should agree with the benchmark merged into the catalog', () => {
// The overlay is empty until the sync runs with an API key configured, so
// this checks the two artifacts cannot drift rather than pinning a count.
- const scored = CATALOG_MODELS.filter(
- (model) => model.benchmark !== undefined,
- );
+ SCORED_CATALOG_MODELS.forEach((model) => {
+ const entry = OVERLAY_MODELS[model.name];
+
+ expect(toBenchmark(entry)).toEqual(model.benchmark);
+ expect(entry.aliases).toContain(model.name);
- scored.forEach((model) => {
- const { aliases, ...overlayEntry } = OVERLAY_MODELS[model.name];
+ const overlayReadings = Object.fromEntries(
+ Object.entries(entry.benchmarkByEffort ?? {}).map(
+ ([effort, reading]) => [effort, withoutAliases(reading)],
+ ),
+ );
+
+ expect(overlayReadings).toEqual(model.benchmarkByEffort ?? {});
+ });
+ });
- expect(overlayEntry).toEqual(model.benchmark);
- expect(aliases).toContain(model.name);
+ it('should score an effort only where the model declares it', () => {
+ CATALOG_MODELS.forEach((model) => {
+ Object.keys(model.benchmarkByEffort ?? {}).forEach((effort) => {
+ expect(model.efforts ?? []).toContain(effort);
+ });
});
});
it('should hold no entry for a model the catalog does not score', () => {
const scoredNames = new Set(
- CATALOG_MODELS.filter((model) => model.benchmark !== undefined).map(
- (model) => model.name,
- ),
+ SCORED_CATALOG_MODELS.map((model) => model.name),
);
Object.keys(OVERLAY_MODELS).forEach((name) => {
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-provider-model-config.schema.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-provider-model-config.schema.spec.ts
new file mode 100644
index 00000000000..9b9cfbde8bc
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/__tests__/ai-provider-model-config.schema.spec.ts
@@ -0,0 +1,101 @@
+import { aiProviderModelConfigSchema } from 'src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema';
+
+const MEASURED_AT = '2026-09-10';
+
+const modelWith = (overrides: Record<string, unknown>) => ({
+ name: 'gpt-x',
+ label: 'GPT X',
+ efforts: ['low', 'high'],
+ benchmark: { intelligenceIndex: 41, effort: 'high', measuredAt: MEASURED_AT },
+ ...overrides,
+});
+
+const issuePaths = (model: Record<string, unknown>): string[] => {
+ const result = aiProviderModelConfigSchema.safeParse(model);
+
+ return result.success
+ ? []
+ : result.error.issues.map((issue) => issue.path.join('.'));
+};
+
+describe('aiProviderModelConfigSchema', () => {
+ it('accepts per-effort readings filed under the effort they were measured at', () => {
+ expect(
+ issuePaths(
+ modelWith({
+ benchmarkByEffort: {
+ low: {
+ intelligenceIndex: 30,
+ effort: 'low',
+ measuredAt: MEASURED_AT,
+ },
+ high: { intelligenceIndex: 41, measuredAt: MEASURED_AT },
+ },
+ }),
+ ),
+ ).toEqual([]);
+ });
+
+ it('rejects a reading filed under another effort than it was measured at', () => {
+ // The registry hands a variant the reading under its own key, so this
+ // would give the low-effort variant a high-effort figure.
+ expect(
+ issuePaths(
+ modelWith({
+ benchmarkByEffort: {
+ low: {
+ intelligenceIndex: 41,
+ effort: 'high',
+ measuredAt: MEASURED_AT,
+ },
+ },
+ }),
+ ),
+ ).toEqual(['benchmarkByEffort.low.effort']);
+ });
+
+ it('rejects a reading for an effort the model does not declare', () => {
+ expect(
+ issuePaths(
+ modelWith({
+ benchmarkByEffort: {
+ max: {
+ intelligenceIndex: 45,
+ effort: 'max',
+ measuredAt: MEASURED_AT,
+ },
+ },
+ }),
+ ),
+ ).toEqual(['benchmarkByEffort.max']);
+ });
+
+ it('rejects per-effort readings on a model that declares no efforts', () => {
+ expect(
+ issuePaths(
+ modelWith({
+ efforts: undefined,
+ benchmarkByEffort: {
+ low: {
+ intelligenceIndex: 30,
+ effort: 'low',
+ measuredAt: MEASURED_AT,
+ },
+ },
+ }),
+ ),
+ ).toEqual(['benchmarkByEffort.low']);
+ });
+
+ it('rejects an effort outside the supported vocabulary', () => {
+ expect(
+ issuePaths(
+ modelWith({
+ benchmarkByEffort: {
+ turbo: { intelligenceIndex: 30, measuredAt: MEASURED_AT },
+ },
+ }),
+ ),
+ ).toContain('benchmarkByEffort.turbo');
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts
index fda1a276710..4ad9d74b6f1 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/services/ai-model-registry.service.ts
@@ -211,8 +211,9 @@ export class AiModelRegistryService {
modelId: variantId,
label: `${baseConfig.label} (${effort})`,
effort,
- // A benchmark describes the effort it was measured at, not this one.
- benchmark: undefined,
+ // A reading describes the effort it was taken at, so the variant gets
+ // the one taken at its effort or none, never the base model's ceiling.
+ benchmark: baseConfig.benchmarkByEffort?.[effort],
});
const baseModelDef = this.providerModelDefCache.get(baseConfig.modelId);
@@ -392,6 +393,7 @@ export class AiModelRegistryService {
supportsReasoning: modelDef.supportsReasoning,
efforts: modelDef.efforts,
benchmark: modelDef.benchmark,
+ benchmarkByEffort: modelDef.benchmarkByEffort,
isDeprecated: modelDef.isDeprecated,
};
}
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema.ts
index eea17794c84..992320cf69b 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema.ts
@@ -1,5 +1,7 @@
import { z } from 'zod';
+import { AI_MODEL_EFFORTS } from 'twenty-shared/ai';
+
// One publisher's reading of a model, taken on one date. An index is only
// meaningful against models scored the same way, so whoever fills this must use
// the same publisher throughout rather than filling gaps from a second. The
@@ -8,5 +10,9 @@ export const aiModelBenchmarkSchema = z.object({
intelligenceIndex: z.number().optional(),
outputTokensPerSecond: z.number().positive().optional(),
costPerTask: z.number().positive().optional(),
+ // The publisher scores a model once per reasoning effort, so a reading is
+ // only comparable with others taken at the same effort. Unset when the
+ // publisher's row names none.
+ effort: z.enum(AI_MODEL_EFFORTS).optional(),
measuredAt: z.string().date(),
});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type.ts
index 9d2b592d5e5..92077d86c75 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type.ts
@@ -29,5 +29,6 @@ export type AiModelConfig = {
// Pinned by a variant id (`model@effort`); unset runs the provider default.
effort?: AiModelEffort;
benchmark?: AiModelBenchmark;
+ benchmarkByEffort?: Partial<Record<AiModelEffort, AiModelBenchmark>>;
isDeprecated?: boolean;
};
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts
index 7ef4352efb6..0262add2236 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/types/ai-provider-model-config.schema.ts
@@ -1,6 +1,7 @@
import { z } from 'zod';
import { AI_MODEL_EFFORTS, DATA_RESIDENCY_KEYS } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
import { AI_MODEL_KINDS } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-model-kinds.const';
import { aiModelBenchmarkSchema } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-benchmark.schema';
@@ -33,6 +34,11 @@ export const aiProviderModelConfigSchema = z
dataResidency: z.enum(DATA_RESIDENCY_KEYS).optional(),
zeroDataRetention: z.boolean().optional(),
benchmark: aiModelBenchmarkSchema.optional(),
+ // Keyed by effort so a pinned variant carries the reading taken at its own
+ // effort instead of the base model's ceiling.
+ benchmarkByEffort: z
+ .partialRecord(z.enum(AI_MODEL_EFFORTS), aiModelBenchmarkSchema)
+ .optional(),
isDeprecated: z.boolean().optional(),
})
.refine(
@@ -44,4 +50,28 @@ export const aiProviderModelConfigSchema = z
message: 'costPerMinute is required for transcription models',
path: ['costPerMinute'],
},
- );
+ )
+ .superRefine((model, context) => {
+ // A variant reads this map by its own effort, so a reading filed under
+ // another effort's key, or under an effort no variant can pin, would hand
+ // it a figure measured elsewhere.
+ for (const [effort, reading] of Object.entries(
+ model.benchmarkByEffort ?? {},
+ )) {
+ if (!(model.efforts ?? []).some((declared) => declared === effort)) {
+ context.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: `benchmarkByEffort.${effort} scores an effort the model does not declare`,
+ path: ['benchmarkByEffort', effort],
+ });
+ }
+
+ if (isDefined(reading?.effort) && reading.effort !== effort) {
+ context.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: `benchmarkByEffort.${effort} carries a reading measured at ${reading.effort}`,
+ path: ['benchmarkByEffort', effort, 'effort'],
+ });
+ }
+ }
+ });