Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

12.4 KB
Raw Blame
#!/usr/bin/env node
/* Copyright 2026 Marimo. All rights reserved. */

import { readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import {
  Document,
  isMap,
  isSeq,
  isScalar,
  parseDocument,
  type YAMLMap,
  type YAMLSeq,
} from "yaml";
import { parseCliArgs } from "./cli.ts";
import type { AiModel, ModelsByProvider } from "./index.ts";
import { ExistingMetadataSchema } from "./metadata.ts";
import { Logger } from "./simple_logger.ts";
import {
  type ExistingByProvider,
  type ExistingEntry,
  MAX_MODELS_PER_PROVIDER,
  mergeModels,
  deriveMetadataUpdates,
  PROVIDER_MAP,
} from "./sources/merge.ts";
import { fetchModelsDev, type ModelsDevApi } from "./sources/models-dev.ts";

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

/**
 * - `append`: keep existing entries, add only what's new (default).
 * - `replace`: overwrite the file with a fresh sync — destructive, for
 *   bootstrapping or regenerating from scratch.
 */
export type SyncMode = "append" | "replace";

interface SyncOptions {
  modelsYamlPath: string;
  /** Pre-loaded api.json for testing; otherwise fetched live. */
  modelsDev?: ModelsDevApi;
  write?: boolean;
  mode?: SyncMode;
  /** Refresh metadata on curated entries without adding models. */
  metadataOnly?: boolean;
  /** Cap on new entries inserted per provider section. */
  maxPerProvider?: number;
  /** Restrict sync to these marimo provider ids (defaults to all). */
  providers?: readonly string[];
}

export interface SyncResult {
  added: number;
  preserved: number;
  updated: number;
  yaml: string;
}

/** Sequence keys rendered in flow style (`[a, b]`) to match the existing file. */
const FLOW_SEQ_KEYS = new Set([
  "roles",
  "capabilities",
  "input_types",
  "output_types",
  "values",
]);

/** Map keys rendered in flow style (`{a: 1, b: 2}`). */
const FLOW_MAP_KEYS = new Set(["cost", "limits"]);

/** YAML serializes `Date` as an ISO timestamp; we want plain `YYYY-MM-DD`. */
function flattenDates(entry: AiModel): Record<string, unknown> {
  const out: Record<string, unknown> = { ...entry };
  for (const [key, value] of Object.entries(out)) {
    if (value instanceof Date) {
      out[key] = value.toISOString().slice(0, 10);
    }
  }
  return out;
}

function parseExistingModels(yamlText: string): ExistingByProvider {
  const doc = parseDocument(yamlText);
  if (doc.contents == null) {
    return {};
  }
  if (!isMap(doc.contents)) {
    throw new Error(
      "Expected a map at the root of models.yml (keyed by provider)",
    );
  }
  const result: Record<string, ExistingEntry[]> = {};
  for (const pair of (doc.contents as YAMLMap).items) {
    const provider = (pair.key as { value?: unknown })?.value;
    if (typeof provider !== "string") {
      continue;
    }
    if (!isSeq(pair.value)) {
      result[provider] = [];
      continue;
    }
    const entries: ExistingEntry[] = [];
    for (const item of (pair.value as YAMLSeq).items) {
      if (!isMap(item)) {
        continue;
      }
      for (const subPair of (item as YAMLMap).items) {
        const key = (subPair.key as { value?: unknown })?.value;
        const value = (subPair.value as { value?: unknown })?.value;
        if (key === "model" && typeof value === "string") {
          entries.push({ model: value });
          break;
        }
      }
    }
    result[provider] = entries;
  }
  return result;
}

/**
 * Build a `YAMLMap` for one entry with the right flow-style array fields.
 */
function buildEntryNode(doc: Document, entry: AiModel): YAMLMap {
  const node = doc.createNode(flattenDates(entry)) as YAMLMap;
  formatCollections(node);
  return node;
}

function formatCollections(node: unknown): void {
  if (isSeq(node)) {
    for (const item of node.items) {
      formatCollections(item);
    }
  } else if (isMap(node)) {
    for (const pair of node.items) {
      const key = isScalar(pair.key) ? pair.key.value : undefined;
      if (typeof key !== "string") {
        continue;
      }
      if (FLOW_SEQ_KEYS.has(key) && isSeq(pair.value)) {
        pair.value.flow = true;
      } else if (FLOW_MAP_KEYS.has(key) && isMap(pair.value)) {
        pair.value.flow = true;
      }
      formatCollections(pair.value);
    }
  }
}

/**
 * Add a `provider: [...]` section to the root map, with a blank line before
 * the key when `spaceBefore` is true (used to separate provider sections).
 */
function addProviderSection(
  doc: Document,
  root: YAMLMap,
  provider: string,
  seq: YAMLSeq,
  spaceBefore: boolean,
): void {
  const keyNode = doc.createNode(provider);
  if (spaceBefore) {
    (keyNode as { spaceBefore?: boolean }).spaceBefore = true;
  }
  root.add({ key: keyNode, value: seq });
}

/**
 * Render a fresh `models.yml` from scratch (replace mode or empty bootstrap).
 */
function renderFresh(entries: ModelsByProvider): string {
  const doc = new Document({});
  const root = doc.contents as YAMLMap;
  for (const [i, [provider, models]] of Object.entries(entries).entries()) {
    const seq = doc.createNode([]) as YAMLSeq;
    for (const [j, model] of models.entries()) {
      const item = buildEntryNode(doc, model);
      if (j > 0) {
        (item as { spaceBefore?: boolean }).spaceBefore = true;
      }
      seq.items.push(item);
    }
    addProviderSection(doc, root, provider, seq, i > 0);
  }
  return doc.toString({ lineWidth: 0, flowCollectionPadding: false });
}

/**
 * Insert new entries into an existing document, creating provider sections if
 * needed, and refresh existing metadata. Preserves curated fields and comments.
 */
function updateDocument(
  yamlText: string,
  newEntries: ModelsByProvider,
  modelsDev: ModelsDevApi,
  providers: readonly string[] | undefined,
): { yaml: string; updated: number } {
  const doc = parseDocument(yamlText);
  if (doc.contents == null) {
    // Bootstrap an empty map at the root.
    doc.contents = doc.createNode({}) as unknown as typeof doc.contents;
  }
  if (!isMap(doc.contents)) {
    throw new Error(
      "Expected a map at the root of models.yml (keyed by provider)",
    );
  }
  const root = doc.contents as unknown as YAMLMap;
  const updated = refreshMetadata(doc, root, modelsDev, providers);

  for (const [provider, models] of Object.entries(newEntries)) {
    if (models.length === 0) {
      continue;
    }
    let seq = findProviderSeq(root, provider);
    if (!seq) {
      seq = doc.createNode([]) as YAMLSeq;
      // New section appended after existing content — always blank-line separated.
      addProviderSection(doc, root, provider, seq, true);
    }
    const originalFirstItem = seq.items[0] as { spaceBefore?: boolean };
    const newItems: YAMLMap[] = [];
    for (const [index, model] of models.entries()) {
      const item = buildEntryNode(doc, model);
      if (index > 0 || originalFirstItem?.spaceBefore) {
        (item as { spaceBefore?: boolean }).spaceBefore = true;
      }
      newItems.push(item);
    }
    if (newItems.length > 0 && seq.items.length > 0) {
      // Preserve a blank line between the prepended block and curated entries.
      originalFirstItem.spaceBefore = true;
    }
    seq.items.unshift(...newItems);
  }

  return {
    yaml:
      updated > 0 || Object.keys(newEntries).length > 0
        ? doc.toString({ lineWidth: 0, flowCollectionPadding: false })
        : yamlText,
    updated,
  };
}

/** Refresh only sourced metadata, preserving curation and unknown values. */
function refreshMetadata(
  doc: Document,
  root: YAMLMap,
  modelsDev: ModelsDevApi,
  providers: readonly string[] | undefined,
): number {
  let updated = 0;
  const providerFilter = providers ? new Set(providers) : null;
  for (const pair of root.items) {
    const provider = isScalar(pair.key) ? pair.key.value : undefined;
    if (typeof provider !== "string" || !isSeq(pair.value)) {
      continue;
    }
    if (providerFilter && !providerFilter.has(provider)) {
      continue;
    }
    const sourceProviders = Object.entries(PROVIDER_MAP)
      .filter(([, target]) => target === provider)
      .map(([source]) => source);
    for (const item of pair.value.items) {
      if (!isMap(item)) {
        continue;
      }
      const modelId = item.get("model");
      if (typeof modelId !== "string") {
        continue;
      }
      // Same first-source precedence as new models; no cross-provider fallback.
      const source = sourceProviders
        .map((id) => modelsDev[id]?.models[modelId])
        .find((model) => model !== undefined);
      if (!source) {
        continue;
      }
      const existing = ExistingMetadataSchema.safeParse(item.toJSON());
      if (!existing.success) {
        Logger.warn(
          `Skipping metadata refresh for ${provider}/${modelId}: ${existing.error.issues[0]?.message}`,
        );
        continue;
      }
      let changed = false;
      const metadataValues = deriveMetadataUpdates(source, existing.data);
      for (const [field, value] of Object.entries(metadataValues)) {
        const previous = item.get(field, true);
        if (JSON.stringify(previous?.toJSON()) === JSON.stringify(value)) {
          continue;
        }
        const metadata = doc.createNode({ [field]: value });
        formatCollections(metadata);
        if (!isMap(metadata)) {
          continue;
        }
        const replacement = metadata.get(field, true);
        if (previous && replacement) {
          replacement.comment = previous.comment;
          replacement.commentBefore = previous.commentBefore;
        }
        item.set(field, replacement);
        changed = true;
      }
      if (changed) {
        updated++;
      }
    }
  }
  return updated;
}

function findProviderSeq(root: YAMLMap, provider: string): YAMLSeq | null {
  for (const pair of root.items) {
    const key = (pair.key as { value?: unknown })?.value;
    if (key === provider && isSeq(pair.value)) {
      return pair.value;
    }
  }
  return null;
}

function countEntries(entries: ModelsByProvider): number {
  let total = 0;
  for (const list of Object.values(entries)) {
    total += list.length;
  }
  return total;
}

export async function syncModels(options: SyncOptions): Promise<SyncResult> {
  const {
    modelsYamlPath,
    write = true,
    mode = "append",
    metadataOnly = false,
    maxPerProvider,
    providers,
  } = options;
  if (metadataOnly && mode === "replace") {
    throw new Error("Metadata-only sync cannot replace the catalog");
  }
  const modelsDev = options.modelsDev ?? (await fetchModelsDev());

  // `replace` mode pretends the file is empty so everything is treated as new.
  const existingText =
    mode === "replace" ? "" : readFileSync(modelsYamlPath, "utf-8");
  const existing = parseExistingModels(existingText);
  const summary = mergeModels(existing, modelsDev, {
    maxPerProvider: metadataOnly ? 0 : maxPerProvider,
    providers,
  });

  const addedCount = countEntries(summary.newEntries);
  const isFresh = mode === "replace" || existingText.trim() === "";
  const { yaml, updated } = isFresh
    ? { yaml: renderFresh(summary.newEntries), updated: 0 }
    : updateDocument(existingText, summary.newEntries, modelsDev, providers);

  if (write && (addedCount > 0 || updated > 0 || mode === "replace")) {
    writeFileSync(modelsYamlPath, yaml);
  }

  return {
    added: addedCount,
    preserved: summary.preservedCount,
    updated,
    yaml,
  };
}

async function main(): Promise<void> {
  try {
    const dataDir = join(__dirname, "../data");
    const args = parseCliArgs(process.argv.slice(2));
    const max = args.maxPerProvider ?? MAX_MODELS_PER_PROVIDER;
    const providers = args.providers?.join(",") ?? "all";
    Logger.info(
      `Fetching models.dev catalog (mode: ${args.mode}, max-per-provider: ${max}, providers: ${providers})...`,
    );

    const result = await syncModels({
      ...args,
      modelsYamlPath: join(dataDir, "models.yml"),
    });

    Logger.info(
      `Sync complete: added ${result.added} new model(s), refreshed metadata for ${result.updated}, preserved ${result.preserved} existing entries.`,
    );
    Logger.info(
      result.added > 0 || result.updated > 0
        ? "Review the diff (git diff packages/llm-info/data/models.yml) and open a PR."
        : "No changes — models.yml is up to date.",
    );
  } catch (error) {
    Logger.error("Sync failed:", error);
    process.exit(1);
  }
}

if (
  process.argv[1] &&
  import.meta.url === pathToFileURL(process.argv[1]).href
) {
  main().catch((error) => {
    Logger.error(error);
    process.exit(1);
  });
}