Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

10.3 KB
Raw Blame
import { getSettings, updateSettingsWith } from './settings.js';
import { ServerError } from '../lib/errorHandler.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
import { modelComparisonImportSchema } from '../lib/validation.js';
import { importModelComparison } from './modelComparison.js';
import { canonicalCatalogModelSlug } from '../lib/comparisonModelScope.js';

export const KNOWN_EFFORTS = ['non-reasoning', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'ultracode'];

const ARTIFICIAL_ANALYSIS_REQUEST_TIMEOUT_MS = 15_000;
const ARTIFICIAL_ANALYSIS_MAX_PAGES = 50;

export function slugify(text) {
  return String(text || '')
    .toLowerCase()
    .replace(/[^a-z0-9.]+/g, '-')
    .replace(/^-+|-+$/g, '');
}

export function parseModelNameAndEffort(rawName) {
  let name = (rawName || '').trim();
  let effort = 'unspecified';
  let configDetail = '';

  const parenMatch = name.match(/^(.*?)\s*\((.*?)\)$/);
  if (parenMatch) {
    name = parenMatch[1].trim();
    const inside = parenMatch[2].trim();
    configDetail = inside;

    const effortMatch = inside.match(/\b(minimal|low|medium|high|xhigh|very_high|max|ultracode|non-reasoning)\b/i);
    if (effortMatch) {
      effort = effortMatch[1].toLowerCase();
      if (effort === 'very_high') effort = 'xhigh';
    } else if (/\bnon-reasoning\b/i.test(inside)) {
      effort = 'non-reasoning';
    } else if (/\breasoning\b/i.test(inside)) {
      effort = 'reasoning';
    }
  }

  const modelSlug = canonicalCatalogModelSlug(slugify(name));
  return { baseName: name, modelSlug, effort, configDetail };
}

// Map of the original 7 reference observations to ensure exact identity parity
const EXISTING_CATALOG_IDENTITIES = new Map([
  ['aa-v4.2-openai-gpt-5.6-sol-max', { provider: 'OpenAI', model: 'gpt-5.6-sol', effort: 'max', configuration: 'Published API model; max reasoning effort' }],
  ['aa-v4.2-openai-gpt-5.6-sol-high', { provider: 'OpenAI', model: 'gpt-5.6-sol', effort: 'high', configuration: 'Published API model; high reasoning effort' }],
  ['aa-v4.2-openai-gpt-5.6-sol-low', { provider: 'OpenAI', model: 'gpt-5.6-sol', effort: 'low', configuration: 'Published API model; low reasoning effort' }],
  ['aa-v4.2-openai-gpt-5.6-terra-max', { provider: 'OpenAI', model: 'gpt-5.6-terra', effort: 'max', configuration: 'Published API model; max reasoning effort' }],
  ['aa-v4.2-openai-gpt-5.6-luna-max', { provider: 'OpenAI', model: 'gpt-5.6-luna', effort: 'max', configuration: 'Published API model; max reasoning effort' }],
  ['aa-v4.2-google-gemini-3.8-flash-high', { provider: 'Google', model: 'gemini-3.8-flash', effort: 'high', configuration: 'Published API model; high reasoning effort' }],
  ['aa-v4.2-anthropic-claude-fable-5-1-max', { provider: 'Anthropic', model: 'claude-fable-5.1', effort: 'max', configuration: 'Adaptive reasoning, max effort, default fallback; published evaluation configuration' }],
]);

export function transformAAModelsToObservations(models, options = {}) {
  const { retrievedAt = new Date().toISOString(), intelligenceIndexVersion } = options;
  const version = String(intelligenceIndexVersion ?? '');
  if (!/^\d+\.\d+(?:\.\d+)*$/.test(version)) {
    throw new ServerError('Artificial Analysis benchmark version is missing or invalid', { status: 502 });
  }
  const benchmark = `Artificial Analysis Intelligence Index v${version}`;
  const observations = [];
  const seenIds = new Set();

  for (const m of models) {
    if (!m || !m.name) continue;
    const { modelSlug, effort, configDetail } = parseModelNameAndEffort(m.name);
    const provider = m.model_creator?.name || 'Unknown';
    const providerSlug = slugify(provider);

    let id = `aa-v${version}-${providerSlug}-${modelSlug}-${effort}`;
    for (const [existId, exist] of EXISTING_CATALOG_IDENTITIES) {
      if (version === '4.2' && exist.provider.toLowerCase() === provider.toLowerCase() &&
          exist.model === modelSlug &&
          exist.effort === effort) {
        id = existId;
        break;
      }
    }

    let finalId = id;
    let counter = 1;
    while (seenIds.has(finalId)) {
      finalId = `${id}-${(m.slug || 'dup').slice(0, 8)}${counter > 1 ? `-${counter}` : ''}`;
      counter++;
    }
    seenIds.add(finalId);

    const sourceUrl = `https://artificialanalysis.ai/models/${m.slug || modelSlug}`;

    const qualityVal = m.evaluations?.artificial_analysis_intelligence_index;
    const quality = Number.isFinite(qualityVal) ? {
      value: Math.round(qualityVal * 10) / 10,
      source: {
        url: sourceUrl,
        retrievedAt,
        methodology: `${benchmark} composite score.`,
      },
    } : null;

    // Preserve only costs Artificial Analysis actually publishes. PortOS no
    // longer derives missing per-effort costs from another model's score curve.
    const costVal = m.artificial_analysis_intelligence_index_cost?.cost_per_task?.total_cost;

    const costPerTask = Number.isFinite(costVal) ? {
      value: Math.round(costVal * 10000) / 10000,
      source: {
        url: sourceUrl,
        retrievedAt,
        methodology: `${benchmark}; cost per task evaluation.`,
      },
    } : null;

    const inputVal = m.pricing?.price_1m_input_tokens;
    const inputPerMillion = Number.isFinite(inputVal) ? {
      value: inputVal,
      source: {
        url: sourceUrl,
        retrievedAt,
        methodology: 'Published USD per million uncached input tokens.',
      },
    } : null;

    const outputVal = m.pricing?.price_1m_output_tokens;
    const outputPerMillion = Number.isFinite(outputVal) ? {
      value: outputVal,
      source: {
        url: sourceUrl,
        retrievedAt,
        methodology: 'Published USD per million output tokens.',
      },
    } : null;

    const respVal = m.performance?.median_end_to_end_response_time_seconds;
    const responseSeconds = Number.isFinite(respVal) ? {
      value: Math.round(respVal * 10) / 10,
      source: {
        url: sourceUrl,
        retrievedAt,
        methodology: 'Artificial Analysis median end-to-end response time (500 answer tokens).',
      },
    } : null;

    const tpsVal = m.performance?.median_output_tokens_per_second;
    const tokensPerSecond = Number.isFinite(tpsVal) ? {
      value: Math.round(tpsVal * 10) / 10,
      source: {
        url: sourceUrl,
        retrievedAt,
        methodology: 'Artificial Analysis median output tokens per second.',
      },
    } : null;

    if (!quality && !costPerTask && !inputPerMillion && !outputPerMillion && !responseSeconds && !tokensPerSecond) {
      continue;
    }

    let finalModel = modelSlug;
    let finalConfiguration = configDetail ? `Published API model; ${configDetail}` : 'Published API model';
    if (EXISTING_CATALOG_IDENTITIES.has(finalId)) {
      finalModel = EXISTING_CATALOG_IDENTITIES.get(finalId).model;
      finalConfiguration = EXISTING_CATALOG_IDENTITIES.get(finalId).configuration;
    }

    observations.push({
      id: finalId,
      provider,
      model: finalModel,
      effort,
      configuration: finalConfiguration.slice(0, 500),
      billing: 'api',
      benchmark,
      quality,
      costPerTask,
      inputPerMillion,
      outputPerMillion,
      reasoningPerMillion: null,
      responseSeconds,
      tokensPerSecond,
      quota: null,
      notes: `Sourced from Artificial Analysis (${m.name}).`,
    });
  }

  return observations;
}

export async function fetchAllArtificialAnalysisModels(apiKey) {
  if (!apiKey) throw new ServerError('Artificial Analysis API key is required', { status: 400 });
  const allModels = [];
  let intelligenceIndexVersion;
  for (let page = 1; page <= ARTIFICIAL_ANALYSIS_MAX_PAGES; page++) {
    let res;
    try {
      res = await fetchWithTimeout(`https://artificialanalysis.ai/api/v2/language/models/free?page=${page}`, {
        headers: { 'x-api-key': apiKey },
      }, ARTIFICIAL_ANALYSIS_REQUEST_TIMEOUT_MS);
    } catch (error) {
      if (error?.name === 'AbortError') {
        throw new ServerError('Artificial Analysis request timed out; retry sync', { status: 502 });
      }
      throw error;
    }
    if (!res.ok) {
      throw new ServerError(`Artificial Analysis API failed (${res.status}): ${res.statusText || 'request rejected'}`, { status: res.status === 401 || res.status === 403 ? 401 : 502 });
    }
    const json = await res.json();
    const version = String(json.intelligence_index_version ?? '');
    if (!Array.isArray(json.data) || !json.data.length || !/^\d+\.\d+(?:\.\d+)*$/.test(version)) {
      throw new ServerError('Artificial Analysis returned invalid model data or benchmark version', { status: 502 });
    }
    if (intelligenceIndexVersion && version !== intelligenceIndexVersion) {
      throw new ServerError('Artificial Analysis benchmark version changed during pagination; retry sync', { status: 502 });
    }
    intelligenceIndexVersion = version;
    allModels.push(...json.data);
    if (!json.pagination?.has_more) break;
    if (page === ARTIFICIAL_ANALYSIS_MAX_PAGES) {
      throw new ServerError('Artificial Analysis pagination exceeded 50 pages; retry sync', { status: 502 });
    }
  }
  return { models: allModels, intelligenceIndexVersion };
}

// Presence only — never the value. The comparison page reads this to decide
// whether the Sync button prompts for a key or just syncs with the stored one.
async function resolveArtificialAnalysisKey(apiKey) {
  const settings = await getSettings();
  return apiKey?.trim() || settings.secrets?.artificialAnalysis?.apiKey || process.env.ARTIFICIAL_ANALYSIS_API_KEY || '';
}

export async function syncArtificialAnalysisCatalog({ apiKey } = {}) {
  const key = await resolveArtificialAnalysisKey(apiKey);
  if (!key) {
    throw new ServerError('No Artificial Analysis API key provided or configured in ARTIFICIAL_ANALYSIS_API_KEY', { status: 400 });
  }
  const { models: rawModels, intelligenceIndexVersion } = await fetchAllArtificialAnalysisModels(key);
  if (apiKey?.trim()) await updateSettingsWith(current => ({
    ...current, secrets: { ...current.secrets, artificialAnalysis: { apiKey: apiKey.trim() } },
  }));
  const observations = transformAAModelsToObservations(rawModels, { intelligenceIndexVersion });
  const validated = modelComparisonImportSchema.parse({
    schemaVersion: 1,
    observations,
  });
  const updated = await importModelComparison(validated);
  return {
    success: true,
    fetched: rawModels.length,
    observations: observations.length,
    total: updated.observations.length,
    catalog: updated,
  };
}