Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
vi.mock('./settings.js', () => ({ getSettings: vi.fn(async () => ({})), updateSettingsWith: vi.fn() }));
import { describe, expect, it, vi } from 'vitest';
import {
slugify,
parseModelNameAndEffort,
transformAAModelsToObservations,
fetchAllArtificialAnalysisModels,
syncArtificialAnalysisCatalog,
} from './artificialAnalysis.js';
import * as modelComparison from './modelComparison.js';
describe('artificialAnalysis service', () => {
describe('slugify', () => {
it('normalizes names into url-safe slugs', () => {
expect(slugify('GPT-5.6 Sol')).toBe('gpt-5.6-sol');
expect(slugify('Claude Opus 5 (Adaptive Reasoning)')).toBe('claude-opus-5-adaptive-reasoning');
expect(slugify(' DeepSeek V4 Pro ')).toBe('deepseek-v4-pro');
expect(slugify('')).toBe('');
expect(slugify(null)).toBe('');
});
});
describe('parseModelNameAndEffort', () => {
it('extracts base model and reasoning effort from parenthetical tags', () => {
expect(parseModelNameAndEffort('GPT-5.6 Sol (low)')).toEqual({
baseName: 'GPT-5.6 Sol',
modelSlug: 'gpt-5.6-sol',
effort: 'low',
configDetail: 'low',
});
expect(parseModelNameAndEffort('GPT-5.6 Terra (max)')).toEqual({
baseName: 'GPT-5.6 Terra',
modelSlug: 'gpt-5.6-terra',
effort: 'max',
configDetail: 'max',
});
expect(parseModelNameAndEffort('Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)')).toEqual({
baseName: 'Claude Opus 5',
modelSlug: 'claude-opus-5',
effort: 'xhigh',
configDetail: 'Adaptive Reasoning, Xhigh Effort',
});
expect(parseModelNameAndEffort('Gemini 3.5 Flash (minimal)')).toEqual({
baseName: 'Gemini 3.5 Flash',
modelSlug: 'gemini-3.5-flash',
effort: 'minimal',
configDetail: 'minimal',
});
expect(parseModelNameAndEffort('MiMo-V2.5-Pro (Non-reasoning)')).toEqual({
baseName: 'MiMo-V2.5-Pro',
modelSlug: 'mimo-v2.5-pro',
effort: 'non-reasoning',
configDetail: 'Non-reasoning',
});
expect(parseModelNameAndEffort('DeepSeek R1 (Reasoning)')).toEqual({
baseName: 'DeepSeek R1',
modelSlug: 'deepseek-r1',
effort: 'reasoning',
configDetail: 'Reasoning',
});
expect(parseModelNameAndEffort('Mistral Large 3')).toEqual({
baseName: 'Mistral Large 3',
modelSlug: 'mistral-large-3',
effort: 'unspecified',
configDetail: '',
});
});
});
describe('transformAAModelsToObservations', () => {
it('transforms raw AA models into schema-compliant observations and preserves reference identities', () => {
const sampleModels = [
{
id: 'aa-uuid-1',
name: 'GPT-5.6 Sol (max)',
slug: 'gpt-5-6-sol',
model_creator: { name: 'OpenAI' },
evaluations: { artificial_analysis_intelligence_index: 51.3 },
artificial_analysis_intelligence_index_cost: { cost_per_task: { total_cost: 1.2495 } },
pricing: { price_1m_input_tokens: 4, price_1m_output_tokens: 20 },
performance: {
median_end_to_end_response_time_seconds: 113.0,
median_output_tokens_per_second: 85.5,
},
},
{
id: 'aa-uuid-2',
name: 'GPT-5.6 Sol (medium)',
slug: 'gpt-5-6-sol-medium',
model_creator: { name: 'OpenAI' },
evaluations: { artificial_analysis_intelligence_index: 46.0 },
artificial_analysis_intelligence_index_cost: { cost_per_task: { total_cost: 0.3671 } },
pricing: { price_1m_input_tokens: 4, price_1m_output_tokens: 20 },
performance: {
median_end_to_end_response_time_seconds: 11.9,
median_output_tokens_per_second: 70.2,
},
},
{
id: 'aa-uuid-3',
name: 'Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)',
slug: 'claude-fable-5-1',
model_creator: { name: 'Anthropic' },
evaluations: { artificial_analysis_intelligence_index: 56.8 },
artificial_analysis_intelligence_index_cost: { cost_per_task: { total_cost: 6.1169 } },
pricing: { price_1m_input_tokens: 10, price_1m_output_tokens: 50 },
performance: {
median_end_to_end_response_time_seconds: 273.9,
median_output_tokens_per_second: 67.1,
},
},
];
const obs = transformAAModelsToObservations(sampleModels, { retrievedAt: '2026-09-05T00:00:00Z', intelligenceIndexVersion: '4.2' });
expect(obs).toHaveLength(3);
const solMax = obs.find(o => o.effort === 'max' && o.model === 'gpt-5.6-sol');
expect(solMax).toBeDefined();
expect(solMax.id).toBe('aa-v4.2-openai-gpt-5.6-sol-max');
expect(solMax.provider).toBe('OpenAI');
expect(solMax.quality.value).toBe(51.3);
expect(solMax.costPerTask.value).toBe(1.2495);
expect(solMax.responseSeconds.value).toBe(113);
expect(solMax.tokensPerSecond.value).toBe(85.5);
// Frozen id, dotted model — max has to join the rest of Fable 5.1's curve.
const fableMax = obs.find(o => o.model === 'claude-fable-5.1');
expect(fableMax).toBeDefined();
expect(fableMax.id).toBe('aa-v4.2-anthropic-claude-fable-5-1-max');
});
it('skips models that have no sourced metrics', () => {
const emptyModel = [{
id: 'empty',
name: 'Empty Model',
slug: 'empty-model',
model_creator: { name: 'Empty' },
evaluations: { artificial_analysis_intelligence_index: null },
artificial_analysis_intelligence_index_cost: null,
pricing: null,
performance: null,
}];
expect(transformAAModelsToObservations(emptyModel, { intelligenceIndexVersion: '4.3' })).toHaveLength(0);
});
});
describe('fetchAllArtificialAnalysisModels', () => {
it('throws 400 when apiKey is missing', async () => {
await expect(fetchAllArtificialAnalysisModels(null)).rejects.toThrow(/API key is required/i);
});
it('handles pagination across multiple pages', async () => {
const originalFetch = globalThis.fetch;
try {
let callCount = 0;
globalThis.fetch = vi.fn().mockImplementation((url) => {
callCount++;
if (callCount === 1) {
return Promise.resolve({
ok: true,
json: async () => ({
intelligence_index_version: 4.3,
data: [{ id: '1', name: 'Page 1 Model', slug: 'p1', model_creator: { name: 'OpenAI' } }],
pagination: { has_more: true },
}),
});
}
return Promise.resolve({
ok: true,
json: async () => ({
intelligence_index_version: 4.3,
data: [{ id: '2', name: 'Page 2 Model', slug: 'p2', model_creator: { name: 'OpenAI' } }],
pagination: { has_more: false },
}),
});
});
const { models, intelligenceIndexVersion } = await fetchAllArtificialAnalysisModels('test-key');
expect(models).toHaveLength(2);
expect(intelligenceIndexVersion).toBe('4.3');
expect(callCount).toBe(2);
} finally {
globalThis.fetch = originalFetch;
}
});
it('maps a hung page request to a redacted 502 timeout', async () => {
vi.useFakeTimers();
const apiKey = 'secret-test-key';
vi.stubGlobal('fetch', vi.fn((_url, { signal }) => new Promise((_resolve, reject) => {
signal.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError')));
})));
try {
const pending = fetchAllArtificialAnalysisModels(apiKey);
vi.advanceTimersByTime(15_000);
const error = await pending.catch(rejection => rejection);
expect(error).toMatchObject({
name: 'ServerError',
status: 502,
message: 'Artificial Analysis request timed out; retry sync',
});
expect(fetch).toHaveBeenCalledTimes(1);
expect(fetch).toHaveBeenCalledWith(
expect.stringContaining('page=1'),
expect.objectContaining({ headers: { 'x-api-key': apiKey } }),
);
expect(error.message).not.toContain(apiKey);
} finally {
vi.useRealTimers();
vi.unstubAllGlobals();
}
});
it('stops after 50 pages when pagination never ends', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
intelligence_index_version: 4.3,
data: [{ id: 'model', name: 'Page Model', slug: 'page-model' }],
pagination: { has_more: true },
}),
});
vi.stubGlobal('fetch', fetchMock);
try {
await expect(fetchAllArtificialAnalysisModels('test-key')).rejects.toMatchObject({
name: 'ServerError',
status: 502,
message: 'Artificial Analysis pagination exceeded 50 pages; retry sync',
});
expect(fetchMock).toHaveBeenCalledTimes(50);
expect(fetchMock).toHaveBeenLastCalledWith(
'https://artificialanalysis.ai/api/v2/language/models/free?page=50',
expect.objectContaining({ headers: { 'x-api-key': 'test-key' } }),
);
} finally {
vi.unstubAllGlobals();
}
});
});
describe('syncArtificialAnalysisCatalog', () => {
it('throws 400 when no API key is provided and env is empty', async () => {
delete process.env.ARTIFICIAL_ANALYSIS_API_KEY;
await expect(syncArtificialAnalysisCatalog({})).rejects.toThrow(/No Artificial Analysis API key provided/i);
});
it.each([
['missing version', { data: [{ name: 'Example' }] }],
['malformed data', { intelligence_index_version: '4.3', data: null }],
['empty data', { intelligence_index_version: '4.3', data: [] }],
])('refuses %s without importing', async (_label, payload) => {
const importSpy = vi.spyOn(modelComparison, 'importModelComparison');
vi.stubGlobal('fetch', vi.fn(async () => ({ ok: true, json: async () => payload })));
try {
await expect(syncArtificialAnalysisCatalog({ apiKey: 'test' })).rejects.toThrow(/invalid/);
expect(importSpy).not.toHaveBeenCalled();
} finally {
vi.unstubAllGlobals();
importSpy.mockRestore();
}
});
it('rejects a benchmark rollover between pages without importing a mixed snapshot', async () => {
const importSpy = vi.spyOn(modelComparison, 'importModelComparison');
const page = version => ({ ok: true, json: async () => ({
intelligence_index_version: version, data: [{ name: 'Example' }], pagination: { has_more: true },
}) });
vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(page('4.2')).mockResolvedValueOnce(page('4.3')));
try {
await expect(syncArtificialAnalysisCatalog({ apiKey: 'test' })).rejects.toThrow(/changed during pagination/);
expect(importSpy).not.toHaveBeenCalled();
} finally {
vi.unstubAllGlobals();
importSpy.mockRestore();
}
});
it('successfully syncs and imports observations', async () => {
const originalFetch = globalThis.fetch;
const importSpy = vi.spyOn(modelComparison, 'importModelComparison').mockResolvedValue({
schemaVersion: 1,
observations: [{ id: 'obs-1' }],
});
try {
globalThis.fetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
intelligence_index_version: 4.3,
data: [{
id: 'm1',
name: 'Synced Model (high)',
slug: 'synced-model',
model_creator: { name: 'Test' },
evaluations: { artificial_analysis_intelligence_index: 40 },
artificial_analysis_intelligence_index_cost: { cost_per_task: { total_cost: 0.5 } },
pricing: { price_1m_input_tokens: 1, price_1m_output_tokens: 2 },
performance: { median_end_to_end_response_time_seconds: 5 },
}],
pagination: { has_more: false },
}),
});
const result = await syncArtificialAnalysisCatalog({ apiKey: 'valid-key' });
expect(result.success).toBe(true);
expect(result.fetched).toBe(1);
expect(result.observations).toBe(1);
expect(importSpy.mock.calls[0][0].observations[0]).toMatchObject({
id: 'aa-v4.3-test-synced-model-high',
benchmark: 'Artificial Analysis Intelligence Index v4.3',
quality: { value: 40, source: { methodology: 'Artificial Analysis Intelligence Index v4.3 composite score.' } },
});
} finally {
globalThis.fetch = originalFetch;
importSpy.mockRestore();
}
});
});
});