Skip to content

The open alternative to Salesforce, designed for AI.

Fix two matcher defects the first populated benchmark sync exposed (#25690)

Félix Malfait2f3f58300931
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts
index ebf57c3c457..e90061e4cfe 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts
@@ -44,6 +44,43 @@ describe('fetchArtificialAnalysisBenchmarks', () => {
expect(record?.outputTokensPerSecond).toBeUndefined();
});
+ it.each([
+ ['best row first', true],
+ ['best row last', false],
+ ])(
+ 'keeps the highest-scoring configuration whatever the order (%s)',
+ async (_label, bestFirst) => {
+ // The rows for one model are the same model at different reasoning
+ // efforts. Picking by how populated a row is scored Sonnet 4.6 at low
+ // effort against Sonnet 5 at max, and the gap read as capability.
+ const lowEffort = {
+ slug: 'claude-sonnet-4-6',
+ name: 'Claude Sonnet 4.6 (Non-reasoning, Low Effort)',
+ evaluations: { artificial_analysis_intelligence_index: 23.3 },
+ median_output_tokens_per_second: 120,
+ cost_per_task: { total_cost: 0.1 },
+ };
+ const maxEffort = {
+ slug: 'claude-sonnet-4-6',
+ name: 'Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)',
+ evaluations: { artificial_analysis_intelligence_index: 36.2 },
+ median_output_tokens_per_second: 80,
+ };
+
+ respondWith(bestFirst ? [maxEffort, lowEffort] : [lowEffort, maxEffort]);
+
+ const record = (await fetchArtificialAnalysisBenchmarks('key')).get(
+ 'claudesonnet46',
+ );
+
+ expect(record?.intelligenceIndex).toBe(36.2);
+ // The whole row travels together, so speed and cost describe the same
+ // configuration the index was measured in.
+ expect(record?.outputTokensPerSecond).toBe(80);
+ expect(record?.costPerTask).toBeUndefined();
+ },
+ );
+
it.each([
['unmeasured row first', true],
['measured row first', false],
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts
index 2ce8ec2acdd..5afeb2f3527 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts
@@ -93,6 +93,50 @@ describe('buildLookupCandidates', () => {
});
});
+ it('does not offer the undated name once the alias resolves to a release', () => {
+ // The leaderboard's bare `mistral-large` row is the Feb '24 model. Letting
+ // it stand in for the release `-latest` currently points at published a
+ // two-year-old score as the current one.
+ const candidates = buildLookupCandidates({
+ modelName: 'mistral-large-latest',
+ siblingModels: MISTRAL_MODELS,
+ });
+
+ expect(candidates).toContain('mistral-large-2512');
+ expect(candidates).not.toContain('mistral-large');
+ });
+
+ it('offers no undated spelling of a resolved full-date release either', () => {
+ // The publisher's undated row is whichever snapshot it last measured, which
+ // is the same trap as the bare rolling name in a longer date format.
+ const candidates = buildLookupCandidates({
+ modelName: 'claude-sonnet-5-latest',
+ siblingModels: {
+ 'claude-sonnet-5-latest': modelsDevModel({
+ id: 'claude-sonnet-5-latest',
+ }),
+ 'claude-sonnet-5-20260101': modelsDevModel({
+ id: 'claude-sonnet-5-20260101',
+ release_date: '2026-01-01',
+ }),
+ },
+ });
+
+ expect(candidates).toContain('claude-sonnet-5-20260101');
+ expect(candidates).not.toContain('claude-sonnet-5');
+ });
+
+ it('falls back to the undated name when no release can be resolved', () => {
+ const candidates = buildLookupCandidates({
+ modelName: 'mistral-large-latest',
+ siblingModels: {
+ 'mistral-large-latest': modelsDevModel({ id: 'mistral-large-latest' }),
+ },
+ });
+
+ expect(candidates).toContain('mistral-large');
+ });
+
it('offers the undated name for a dated snapshot', () => {
expect(
buildLookupCandidates({
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/build-lookup-candidates.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-lookup-candidates.util.ts
index f357af829e1..1807c88aa66 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/build-lookup-candidates.util.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-lookup-candidates.util.ts
@@ -55,13 +55,15 @@ export const buildLookupCandidates = ({
const resolved = resolveRollingAlias(modelName, siblingModels);
if (isDefined(resolved)) {
- candidates.push(resolved, resolved.replace(DATE_SUFFIX, ''));
+ // Once the release an alias points at is known, only a row naming that
+ // release will do. Every undated spelling is some other release the
+ // publisher happens to have measured: `mistral-large` on the leaderboard is
+ // the Feb '24 model, and standing in for `mistral-large-2512` it published a
+ // two-year-old score as current.
+ return [...new Set([...candidates, resolved])];
}
- // Last: an undated `mistral-large` row in the index is whichever release the
- // publisher last measured, so it must not win over the release the alias
- // currently resolves to.
- candidates.push(modelName.replace(ROLLING_SUFFIX, ''));
-
- return [...new Set(candidates)];
+ // No dated release to point at, so the bare name is the only thing left and
+ // whichever release the publisher measured is the best answer available.
+ return [...new Set([...candidates, modelName.replace(ROLLING_SUFFIX, '')])];
};
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts
index fe241b99b12..25436b5bf77 100644
--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts
+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts
@@ -58,6 +58,26 @@ const informationScore = (record: BenchmarkRecord): number =>
record.costPerTask,
].filter(isDefined).length;
+// The rows for one model are the same model at different reasoning efforts, and
+// the publisher spells the effort only inside a display name. Taking the
+// best-scoring row gives every model its ceiling, measured the same way, which
+// is the whole reason for standing on one publisher. Picking by how populated a
+// row happened to be scored Sonnet 4.6 at low effort against Sonnet 5 at max,
+// and the gap read as a capability difference.
+const preferOver = (
+ candidate: BenchmarkRecord,
+ existing: BenchmarkRecord,
+): boolean => {
+ const candidateIndex = candidate.intelligenceIndex ?? -1;
+ const existingIndex = existing.intelligenceIndex ?? -1;
+
+ if (candidateIndex !== existingIndex) {
+ return candidateIndex > existingIndex;
+ }
+
+ return informationScore(candidate) > informationScore(existing);
+};
+
export const fetchArtificialAnalysisBenchmarks = async (
apiKey: string,
): Promise<BenchmarkIndex> => {
@@ -120,8 +140,7 @@ export const fetchArtificialAnalysisBenchmarks = async (
if (
key.length > 0 &&
- (!isDefined(existing) ||
- informationScore(record) > informationScore(existing))
+ (!isDefined(existing) || preferOver(record, existing))
) {
index.set(key, record);
}