Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

8.6 KB
Raw Blame
import { isEmbeddingModel, type EmbeddingModel } from "./model-overlay.ts";
import type { ModelGatewayTransportConfig } from "./provider-endpoints.ts";
import { GATEWAY_MODEL_PREFIX, GATEWAY_PROVIDER, setGatewayModels, type GatewayModel } from "./gateway-models.ts";
import { modelOfferedInWebui, selectableBaseModels } from "./pi-models.ts";

const MAX_BYTES = 2 * 1024 * 1024;
const MAX_MODELS = 1_000;
const TTL_MS = 5 * 60_000;
const RETRY_MS = 30_000;
const STALE_MS = 15 * 60_000;

function transientFailure(error: unknown): boolean {
  const status = (error as { status?: unknown } | null)?.status;
  if (typeof status === "number") return status === 429 || status >= 500;
  const name = (error as { name?: unknown } | null)?.name;
  return name === "TimeoutError" || name === "AbortError" || error instanceof TypeError;
}

function record(value: unknown): Record<string, unknown> | undefined {
  return value !== null && typeof value === "object" && !Array.isArray(value)
    ? (value as Record<string, unknown>)
    : undefined;
}

function modelId(value: unknown): value is string {
  return typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,191}$/.test(value);
}

function tokens(value: unknown): number | undefined {
  return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined;
}

function price(value: unknown): number | undefined {
  return typeof value === "number" && Number.isFinite(value) && value >= 0 && Number.isFinite(value * 1_000_000)
    ? value * 1_000_000
    : undefined;
}

function parseModel(value: unknown, allowed: Set<string>, baseUrl: string): GatewayModel | EmbeddingModel | undefined {
  const info = record(value);
  if (!info || !modelId(info.model_group) || !allowed.has(info.model_group)) return undefined;
  if (info.mode === "embedding") {
    return {
      kind: "embedding",
      id: GATEWAY_MODEL_PREFIX + info.model_group,
      name: info.model_group,
      provider: GATEWAY_PROVIDER,
      baseUrl,
    };
  }
  if (info.mode !== "chat" || info.supports_function_calling !== true) return undefined;
  const contextWindow = tokens(info.max_input_tokens);
  const maxTokens = tokens(info.max_output_tokens);
  const input = price(info.input_cost_per_token);
  const output = price(info.output_cost_per_token);
  if (!contextWindow || !maxTokens || maxTokens >= contextWindow || input === undefined || output === undefined)
    return undefined;
  const providers = Array.isArray(info.providers) ? info.providers : [];
  const anthropic = providers.length > 0 && providers.every((provider) => provider === "anthropic");
  const openai = providers.length > 0 && providers.every((provider) => provider === "openai" || provider === "azure");
  const pdf =
    providers.length > 0 &&
    providers.every((provider) => ["anthropic", "openai", "azure", "gemini", "vertex_ai"].includes(String(provider)));
  let api: GatewayModel["api"] = "openai-completions";
  if (anthropic) api = "anthropic-messages";
  else if (openai) api = "openai-responses";
  let documentInput: GatewayModel["documentInput"];
  if (openai) documentInput = "files";
  else if (pdf) documentInput = "pdf";
  return {
    documentInput,
    id: GATEWAY_MODEL_PREFIX + info.model_group,
    name: info.model_group,
    provider: GATEWAY_PROVIDER,
    api,
    baseUrl: anthropic ? baseUrl.replace(/\/v1$/, "") : baseUrl,
    reasoning: (anthropic || openai) && info.supports_reasoning === true,
    input: info.supports_vision === true ? ["text", "image"] : ["text"],
    contextWindow,
    maxTokens,
    cost: {
      input,
      output,
      cacheRead: price(info.cache_read_input_token_cost) ?? input,
      cacheWrite: price(info.cache_creation_input_token_cost) ?? input,
    },
    compat: {
      supportsDeveloperRole: false,
      supportsStore: false,
      supportsStrictMode: false,
      supportsStrictTools: false,
      forceAdaptiveThinking: anthropic && info.supports_adaptive_thinking === true,
      supportsLongCacheRetention: false,
      supportsReasoningEffort:
        Array.isArray(info.supported_openai_params) && info.supported_openai_params.includes("reasoning_effort"),
      supportsUsageInStreaming: true,
      sendSessionAffinityHeaders: true,
      maxTokensField:
        Array.isArray(info.supported_openai_params) && info.supported_openai_params.includes("max_completion_tokens")
          ? "max_completion_tokens"
          : "max_tokens",
    },
  };
}

async function readCatalog(
  url: string,
  config: ModelGatewayTransportConfig,
  fetcher: typeof fetch,
): Promise<unknown[]> {
  const response = await fetcher(url, {
    headers: { [config.apiKeyHeader]: config.apiKey },
    signal: AbortSignal.timeout(5_000),
    redirect: "error",
  });
  if (!response.ok)
    throw Object.assign(new Error(`Gateway catalog returned HTTP ${response.status}`), { status: response.status });
  if (!response.body || Number(response.headers.get("content-length")) > MAX_BYTES)
    throw new Error("Gateway catalog exceeds size limit");
  const reader = response.body.getReader();
  const decoder = new TextDecoder();
  let bytes = 0;
  let body = "";
  try {
    while (true) {
      const chunk = await reader.read();
      if (chunk.done) break;
      bytes += chunk.value.byteLength;
      if (bytes > MAX_BYTES) throw new Error("Gateway catalog exceeds size limit");
      body += decoder.decode(chunk.value, { stream: true });
    }
    body += decoder.decode();
  } finally {
    void reader.cancel().catch(() => {});
    reader.releaseLock();
  }
  const data = record(JSON.parse(body))?.data;
  if (!Array.isArray(data) || data.length > MAX_MODELS) throw new Error("Invalid gateway catalog");
  return data;
}

export function createGatewayCatalog(
  config: ModelGatewayTransportConfig,
  fetcher: typeof fetch = fetch,
  now: () => number = Date.now,
) {
  const root = config.url.replace(/\/v1$/, "");
  const baseUrl = `${root}/v1`;
  let routes: Readonly<Record<string, string>> = { ...config.models };
  let discovered = false;
  let expiresAt = 0;
  let retryAt = 0;
  let lastSuccessAt = 0;
  let inFlight: Promise<void> | undefined;
  const transport: ModelGatewayTransportConfig = {
    ...config,
    reservedModelIds: new Set(Object.keys(config.models)),
    refresh,
    get models() {
      return routes;
    },
  };
  async function refresh(): Promise<void> {
    if (inFlight) return inFlight;
    if (now() < retryAt) return;
    inFlight = (async () => {
      try {
        const [listing, metadata] = await Promise.all([
          readCatalog(`${baseUrl}/models`, config, fetcher),
          readCatalog(`${root}/model_group/info`, config, fetcher),
        ]);
        const allowed = new Set(
          listing.flatMap((entry) => {
            const id = record(entry)?.id;
            if (!modelId(id)) throw new Error("Invalid gateway model id");
            return [id];
          }),
        );
        const models = metadata.flatMap((entry) => {
          const model = parseModel(entry, allowed, baseUrl);
          return model ? [model] : [];
        });
        const next: Record<string, string> = Object.fromEntries(
          models.map((model) => [model.id, model.id.slice(GATEWAY_MODEL_PREFIX.length)]),
        );
        for (const [id, target] of Object.entries(config.models)) {
          if (allowed.has(target) && !models.some((m) => m.id === GATEWAY_MODEL_PREFIX + target && isEmbeddingModel(m)))
            next[id] = target;
        }
        routes = next;
        discovered = true;
        expiresAt = now() + TTL_MS;
        retryAt = expiresAt;
        lastSuccessAt = now();
        const offeredAliases = new Set(
          selectableBaseModels()
            .filter(({ id }) => modelOfferedInWebui(id))
            .map(({ id }) => id),
        );
        setGatewayModels(
          models,
          Object.entries(config.models).flatMap(([id, target]) =>
            !id.startsWith(GATEWAY_MODEL_PREFIX) && offeredAliases.has(id) && next[id]
              ? [GATEWAY_MODEL_PREFIX + target]
              : [],
          ),
        );
      } catch (error) {
        retryAt = now() + RETRY_MS;
        if (discovered && transientFailure(error) && now() - lastSuccessAt < STALE_MS) {
          console.warn("[model] Gateway model discovery failed; keeping last known models and retrying in 30 seconds");
          return;
        }
        if (discovered || Object.keys(config.models).length === 0) {
          routes = {};
          expiresAt = 0;
          setGatewayModels([]);
          console.warn("[model] Gateway model discovery unavailable; retrying in 30 seconds");
        }
      }
    })().finally(() => {
      inFlight = undefined;
    });
    return inFlight;
  }
  return { transport, refresh };
}