The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Fix two matcher defects the first populated benchmark sync exposed (#25690)
Félix Malfait2f3f58300931
diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.tsindex ebf57c3c457..e90061e4cfe 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/fetch-artificial-analysis.spec.ts@@ -44,6 +44,43 @@ describe('fetchArtificialAnalysisBenchmarks', () => {expect(record?.outputTokensPerSecond).toBeUndefined();});+ it.each([+ ['best row first', true],+ ['best row last', false],+ ])(+ 'keeps the highest-scoring configuration whatever the order (%s)',+ async (_label, bestFirst) => {+ // The rows for one model are the same model at different reasoning+ // efforts. Picking by how populated a row is scored Sonnet 4.6 at low+ // effort against Sonnet 5 at max, and the gap read as capability.+ const lowEffort = {+ slug: 'claude-sonnet-4-6',+ name: 'Claude Sonnet 4.6 (Non-reasoning, Low Effort)',+ evaluations: { artificial_analysis_intelligence_index: 23.3 },+ median_output_tokens_per_second: 120,+ cost_per_task: { total_cost: 0.1 },+ };+ const maxEffort = {+ slug: 'claude-sonnet-4-6',+ name: 'Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)',+ evaluations: { artificial_analysis_intelligence_index: 36.2 },+ median_output_tokens_per_second: 80,+ };++ respondWith(bestFirst ? [maxEffort, lowEffort] : [lowEffort, maxEffort]);++ const record = (await fetchArtificialAnalysisBenchmarks('key')).get(+ 'claudesonnet46',+ );++ expect(record?.intelligenceIndex).toBe(36.2);+ // The whole row travels together, so speed and cost describe the same+ // configuration the index was measured in.+ expect(record?.outputTokensPerSecond).toBe(80);+ expect(record?.costPerTask).toBeUndefined();+ },+ );+it.each([['unmeasured row first', true],['measured row first', false],diff --git a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.tsindex 2ce8ec2acdd..5afeb2f3527 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/__tests__/match-benchmarks.spec.ts@@ -93,6 +93,50 @@ describe('buildLookupCandidates', () => {});});+ it('does not offer the undated name once the alias resolves to a release', () => {+ // The leaderboard's bare `mistral-large` row is the Feb '24 model. Letting+ // it stand in for the release `-latest` currently points at published a+ // two-year-old score as the current one.+ const candidates = buildLookupCandidates({+ modelName: 'mistral-large-latest',+ siblingModels: MISTRAL_MODELS,+ });++ expect(candidates).toContain('mistral-large-2512');+ expect(candidates).not.toContain('mistral-large');+ });++ it('offers no undated spelling of a resolved full-date release either', () => {+ // The publisher's undated row is whichever snapshot it last measured, which+ // is the same trap as the bare rolling name in a longer date format.+ const candidates = buildLookupCandidates({+ modelName: 'claude-sonnet-5-latest',+ siblingModels: {+ 'claude-sonnet-5-latest': modelsDevModel({+ id: 'claude-sonnet-5-latest',+ }),+ 'claude-sonnet-5-20260101': modelsDevModel({+ id: 'claude-sonnet-5-20260101',+ release_date: '2026-01-01',+ }),+ },+ });++ expect(candidates).toContain('claude-sonnet-5-20260101');+ expect(candidates).not.toContain('claude-sonnet-5');+ });++ it('falls back to the undated name when no release can be resolved', () => {+ const candidates = buildLookupCandidates({+ modelName: 'mistral-large-latest',+ siblingModels: {+ 'mistral-large-latest': modelsDevModel({ id: 'mistral-large-latest' }),+ },+ });++ expect(candidates).toContain('mistral-large');+ });+it('offers the undated name for a dated snapshot', () => {expect(buildLookupCandidates({diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/build-lookup-candidates.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-lookup-candidates.util.tsindex f357af829e1..1807c88aa66 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/build-lookup-candidates.util.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/build-lookup-candidates.util.ts@@ -55,13 +55,15 @@ export const buildLookupCandidates = ({const resolved = resolveRollingAlias(modelName, siblingModels);if (isDefined(resolved)) {- candidates.push(resolved, resolved.replace(DATE_SUFFIX, ''));+ // Once the release an alias points at is known, only a row naming that+ // release will do. Every undated spelling is some other release the+ // publisher happens to have measured: `mistral-large` on the leaderboard is+ // the Feb '24 model, and standing in for `mistral-large-2512` it published a+ // two-year-old score as current.+ return [...new Set([...candidates, resolved])];}- // Last: an undated `mistral-large` row in the index is whichever release the- // publisher last measured, so it must not win over the release the alias- // currently resolves to.- candidates.push(modelName.replace(ROLLING_SUFFIX, ''));-- return [...new Set(candidates)];+ // No dated release to point at, so the bare name is the only thing left and+ // whichever release the publisher measured is the best answer available.+ return [...new Set([...candidates, modelName.replace(ROLLING_SUFFIX, '')])];};diff --git a/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts b/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.tsindex fe241b99b12..25436b5bf77 100644--- a/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts+++ b/packages/twenty-server/scripts/ai-catalog-sync/utils/fetch-artificial-analysis-benchmarks.util.ts@@ -58,6 +58,26 @@ const informationScore = (record: BenchmarkRecord): number =>record.costPerTask,].filter(isDefined).length;+// The rows for one model are the same model at different reasoning efforts, and+// the publisher spells the effort only inside a display name. Taking the+// best-scoring row gives every model its ceiling, measured the same way, which+// is the whole reason for standing on one publisher. Picking by how populated a+// row happened to be scored Sonnet 4.6 at low effort against Sonnet 5 at max,+// and the gap read as a capability difference.+const preferOver = (+ candidate: BenchmarkRecord,+ existing: BenchmarkRecord,+): boolean => {+ const candidateIndex = candidate.intelligenceIndex ?? -1;+ const existingIndex = existing.intelligenceIndex ?? -1;++ if (candidateIndex !== existingIndex) {+ return candidateIndex > existingIndex;+ }++ return informationScore(candidate) > informationScore(existing);+};+export const fetchArtificialAnalysisBenchmarks = async (apiKey: string,): Promise<BenchmarkIndex> => {@@ -120,8 +140,7 @@ export const fetchArtificialAnalysisBenchmarks = async (if (key.length > 0 &&- (!isDefined(existing) ||- informationScore(record) > informationScore(existing))+ (!isDefined(existing) || preferOver(record, existing))) {index.set(key, record);}