Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

14.4 KB
Raw Blame
/**
 * Media generation for the agent, its commands and the apps (ADR-042).
 *
 * Which model makes an image, in order:
 *   1. the one asked for (a tool argument, /image --model, a desktop pick)
 *   2. the owner's default for images, then its fallbacks (media-prefs.json)
 *   3. automatic: the best image model found on the configured providers,
 *      the current chat provider's first
 * A model that fails hands over to the next (at most three are tried).
 */
import { mkdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { freeName } from '../artifacts/output-dir.js';
import { logger } from '../utils/logger.js';
import { imageModels, knownImageModels, musicModels, parseRef, videoModels, type MediaKind, type MediaModel, type MediaSource } from './catalog.js';
import { generateMusicWith, generateVideoWith, type MusicRequest, type VideoRequest } from './video.js';
import { generateWith, ImageGenerationError, type ImageRequest } from './image.js';
import { loadMediaPrefs } from './prefs.js';

export interface MediaHost {
  /** The configured providers, in the registry's order. */
  sources(): MediaSource[];
  /** The provider the chat is using now (preferred for automatic picks). */
  currentProvider(): string | undefined;
}

export interface ImageJob extends Omit<ImageRequest, 'signal'> {
  /** "provider::model" to use this one only. */
  model?: string;
  signal?: AbortSignal;
}

export interface ImageJobResult {
  files: string[];
  model: MediaModel;
  /** Models that failed before this one, with why. */
  failed: Array<{ ref: string; error: string }>;
  revisedPrompt?: string;
  note?: string;
  ms: number;
}

/**
 * Models the provider says this account can't use (a free tier with no
 * quota, no billing): skipped by automatic picks for a day instead of
 * failing first on every request. A model asked for by name is still tried.
 */
const NO_QUOTA = /limit:\s*0\b|free tier|upgrade your (tier|plan)|billing (is )?(not|required)|insufficient[_ ]quota|not available (on|for) your (plan|tier)|payment required/i;
const NO_QUOTA_MS = 24 * 60 * 60 * 1000;
const unusable = new Map<string, { until: number; why: string }>();

export function noteMediaFailure(ref: string, error: string, now = Date.now()): void {
  if (NO_QUOTA.test(error)) unusable.set(ref, { until: now + NO_QUOTA_MS, why: 'no quota on this account' });
}

/** Why a model is being skipped right now, if it is. */
export function unusableReason(ref: string, now = Date.now()): string | undefined {
  const u = unusable.get(ref);
  if (!u) return undefined;
  if (u.until < now) { unusable.delete(ref); return undefined; }
  return u.why;
}

/** Test hook. */
export function resetUnusable(): void {
  unusable.clear();
}

/** Better-known models first when picking automatically, per kind. */
const PREFERENCE: Record<string, RegExp[]> = {
  image: [/gpt-image|chatgpt-image/i, /nano-banana|gemini-[\w.-]*-image/i, /flux/i, /seedream/i, /imagen/i, /grok-imagine|grok-2-image/i, /qwen-image|z-image/i, /dall-e-3/i],
  video: [/omni/i, /veo/i, /sora/i, /kling/i, /seedance/i, /hailuo|minimax/i, /runway|gen-?4/i, /luma|ray/i, /wan/i, /ltx/i],
  music: [/lyria-3\.5|lyria-3-pro/i, /music_v2_5/i, /lyria/i, /music_v2/i, /music/i],
  speech: [],
};

function rank(m: MediaModel): number {
  const list = PREFERENCE[m.kind] ?? [];
  const i = list.findIndex((re) => re.test(m.model));
  return i < 0 ? list.length : i;
}

/** The automatic pick order among known models. */
export function autoOrder(models: MediaModel[], currentProvider?: string): MediaModel[] {
  return [...models].sort((a, b) => {
    const pa = a.provider === currentProvider ? 0 : 1;
    const pb = b.provider === currentProvider ? 0 : 1;
    if (pa !== pb) return pa - pb;
    const r = rank(a) - rank(b);
    if (r) return r;
    // Newer versions tend to sort later by name ("gpt-image-1.5" > "gpt-image-1").
    return b.model.localeCompare(a.model, undefined, { numeric: true });
  });
}

export interface ImageSetup {
  /** The owner's default, if set (and whether it is still offered). */
  default?: { ref: string; available: boolean };
  /** What "automatic" would use now. */
  auto?: MediaModel;
  models: MediaModel[];
}

export class MediaService {
  constructor(private readonly host: MediaHost, private readonly fetchImpl: typeof fetch = globalThis.fetch) {}

  /** A configured provider's endpoint and auth (fresh), for other callers that need it. */
  async endpointFor(name: string): Promise<import('../providers/base.js').MediaEndpoint | undefined> {
    return this.host.sources().find((s) => s.name === name)?.endpoint().catch(() => undefined);
  }

  /** The providers, for the speech service (same sources as images). */
  sourcesForSpeech(): () => MediaSource[] {
    return () => this.host.sources();
  }

  async imageModels(opts: { refresh?: boolean } = {}): Promise<MediaModel[]> {
    return imageModels(this.host.sources(), { refresh: opts.refresh, fetchImpl: this.fetchImpl });
  }

  /** Known without network: for tool descriptions and quick checks. */
  knownImageModels(): MediaModel[] {
    return knownImageModels(this.host.sources());
  }

  async models(kind: Exclude<MediaKind, 'speech'>, opts: { refresh?: boolean } = {}): Promise<MediaModel[]> {
    const lister = kind === 'video' ? videoModels : kind === 'music' ? musicModels : imageModels;
    return lister(this.host.sources(), { refresh: opts.refresh, fetchImpl: this.fetchImpl });
  }

  async setup(kind: Exclude<MediaKind, 'speech'>, opts: { refresh?: boolean } = {}): Promise<ImageSetup> {
    const models = await this.models(kind, opts);
    const primary = loadMediaPrefs()[kind]?.primary;
    return {
      ...(primary ? { default: { ref: primary, available: models.some((m) => m.ref === primary) } } : {}),
      auto: autoOrder(models, this.host.currentProvider()).find((m) => !unusableReason(m.ref)),
      models,
    };
  }

  async imageSetup(opts: { refresh?: boolean } = {}): Promise<ImageSetup> {
    const models = await this.imageModels(opts);
    const primary = loadMediaPrefs().image?.primary;
    return {
      ...(primary ? { default: { ref: primary, available: models.some((m) => m.ref === primary) } } : {}),
      auto: autoOrder(models, this.host.currentProvider()).find((m) => !unusableReason(m.ref)),
      models,
    };
  }

  /** The models to try, in order (at most three). */
  async imageChain(requested?: string): Promise<MediaModel[]> {
    return this.chain('image', requested);
  }

  async chain(kind: Exclude<MediaKind, 'speech'>, requested?: string): Promise<MediaModel[]> {
    const models = await this.models(kind);
    const byRef = new Map(models.map((m) => [m.ref, m]));
    const fromRef = (ref: string): MediaModel | undefined => {
      const known = byRef.get(ref);
      if (known) return known;
      // A model the owner named that discovery didn't list: try it through its provider's Images API.
      const p = parseRef(ref);
      if (!p || !this.host.sources().some((s) => s.name === p.provider) || kind === 'music') return undefined;
      return { ref, provider: p.provider, model: p.model, kind, backend: kind === 'video' ? 'openai-videos' : 'openai-images', label: p.model, group: p.provider, canEdit: true };
    };
    if (requested) {
      const exact = fromRef(requested) ?? models.find((m) => m.model === requested || m.label.toLowerCase() === requested.toLowerCase());
      // A service by name ("elevenlabs", "Eleven Labs", "lyria", "veo"): its models only, never another service's.
      return exact ? [exact] : byService(models, requested).slice(0, 3);
    }
    const prefs = loadMediaPrefs()[kind];
    const chain: MediaModel[] = [];
    for (const ref of [prefs?.primary, ...(prefs?.fallbacks ?? [])]) {
      const m = ref ? fromRef(ref) : undefined;
      if (m && !chain.some((c) => c.ref === m.ref)) chain.push(m);
    }
    for (const m of autoOrder(models, this.host.currentProvider())) {
      if (chain.length >= 3) break;
      if (!chain.some((c) => c.ref === m.ref) && !unusableReason(m.ref)) chain.push(m);
    }
    return chain.slice(0, 3);
  }

  /** A video: Gemini Omni or an OpenAI Videos API gateway; can take minutes (run it as a job). */
  generateVideo(req: VideoRequest & { model?: string }, outDir: string): Promise<MediaJobResult> {
    return runChain(this, this.host, 'video', req, outDir, (m, ep) => generateVideoWith(m, ep, req, this.fetchImpl));
  }

  /** A song or clip: Gemini Lyria or ElevenLabs Music. */
  generateMusic(req: MusicRequest & { model?: string }, outDir: string): Promise<MediaJobResult> {
    return runChain(this, this.host, 'music', req, outDir, (m, ep) => generateMusicWith(m, ep, req, this.fetchImpl));
  }

  /** Generate, save into `outDir`, and say which model made it. */
  async generateImage(job: ImageJob, outDir: string): Promise<ImageJobResult> {
    const started = Date.now();
    let chain = await this.imageChain(job.model);
    if (job.references?.length) {
      const editors = chain.filter((m) => m.canEdit);
      chain = editors.length ? editors : chain;
    }
    if (!chain.length) {
      throw new ImageGenerationError(job.model
        ? `No image model "${job.model}" on the configured providers.`
        : 'No image model is set up. Add a provider that makes images (OpenAI, OpenRouter, LiteLLM, AI/ML API, Mercury Cloud…) and pick one with /media.');
    }
    const failed: ImageJobResult['failed'] = [];
    for (const m of chain) {
      if (job.signal?.aborted) break;
      const src = this.host.sources().find((s) => s.name === m.provider);
      const ep = await src?.endpoint().catch(() => undefined);
      if (!ep) { failed.push({ ref: m.ref, error: 'provider not reachable' }); continue; }
      try {
        const result = await generateWith(m, ep, job, this.fetchImpl);
        mkdirSync(outDir, { recursive: true });
        const stem = slug(job.prompt);
        const files = result.images.map((img, i) => {
          const ext = img.mime === 'image/jpeg' ? 'jpg' : img.mime === 'image/webp' ? 'webp' : img.mime === 'image/gif' ? 'gif' : 'png';
          const name = freeName(outDir, `${stem}${result.images.length > 1 ? `-${i + 1}` : ''}.${ext}`);
          const file = join(outDir, name);
          writeFileSync(file, img.bytes);
          return file;
        });
        return { files, model: m, failed, ms: Date.now() - started, ...(result.revisedPrompt ? { revisedPrompt: result.revisedPrompt } : {}), ...(result.note ? { note: result.note } : {}) };
      } catch (err) {
        if (job.signal?.aborted) throw err;
        const error = err instanceof Error ? err.message : String(err);
        logger.warn({ model: m.ref, error }, 'Image generation failed; trying the next model');
        noteMediaFailure(m.ref, error);
        failed.push({ ref: m.ref, error });
      }
    }
    throw new ImageGenerationError(`Image generation failed: ${failed.map((f) => `${f.ref} (${f.error})`).join('; ')}`);
  }
}

/** A video or song, saved into `outDir`. */
export interface MediaJobResult {
  file: string;
  model: MediaModel;
  failed: Array<{ ref: string; error: string }>;
  note?: string;
  ms: number;
}

const EXT: Record<string, string> = { 'video/mp4': 'mp4', 'video/webm': 'webm', 'video/quicktime': 'mov', 'audio/mpeg': 'mp3', 'audio/mp3': 'mp3', 'audio/wav': 'wav', 'audio/x-wav': 'wav', 'audio/ogg': 'ogg', 'audio/flac': 'flac' };

/** Try each model in turn; save what the first one that works makes. */
export async function runChain<R extends { signal?: AbortSignal }>(
  service: MediaService,
  host: MediaHost,
  kind: 'video' | 'music',
  req: R & { prompt: string; model?: string; references?: string[] },
  outDir: string,
  call: (m: MediaModel, ep: NonNullable<Awaited<ReturnType<MediaSource['endpoint']>>>) => Promise<{ bytes: Buffer; mime: string; note?: string }>,
): Promise<MediaJobResult> {
  const started = Date.now();
  let chain = await service.chain(kind, req.model);
  if (req.references?.length) {
    const editors = chain.filter((m) => m.canEdit);
    chain = editors.length ? editors : chain;
  }
  if (!chain.length) {
    throw new ImageGenerationError(req.model
      ? `No ${kind} model "${req.model}" on the configured providers.`
      : kind === 'video'
        ? 'No video model is set up. A Gemini key (Settings → Voice) gives Gemini Omni video; gateways such as LiteLLM or Mercury Cloud can serve others. Pick one with /media.'
        : 'No music model is set up. A Gemini key (Lyria) or an ElevenLabs key (Settings → Voice) makes music. Pick one with /media.');
  }
  const failed: MediaJobResult['failed'] = [];
  for (const m of chain) {
    if (req.signal?.aborted) break;
    const ep = await host.sources().find((s) => s.name === m.provider)?.endpoint().catch(() => undefined);
    if (!ep) { failed.push({ ref: m.ref, error: 'provider not reachable' }); continue; }
    try {
      const out = await call(m, ep);
      mkdirSync(outDir, { recursive: true });
      const file = join(outDir, freeName(outDir, `${slug(req.prompt)}.${EXT[out.mime] ?? (kind === 'video' ? 'mp4' : 'mp3')}`));
      writeFileSync(file, out.bytes);
      return { file, model: m, failed, ms: Date.now() - started, ...(out.note ? { note: out.note } : {}) };
    } catch (err) {
      if (req.signal?.aborted) throw err;
      const error = err instanceof Error ? err.message : String(err);
      logger.warn({ model: m.ref, kind, error }, 'Media generation failed; trying the next model');
      noteMediaFailure(m.ref, error);
      failed.push({ ref: m.ref, error });
    }
  }
  throw new ImageGenerationError(`${kind === 'video' ? 'Video' : 'Music'} generation failed: ${failed.map((f) => `${f.ref} (${f.error})`).join('; ')}`);
}

/** "A red fox in snow, watercolor" → "red-fox-in-snow-watercolor". */
export function slug(prompt: string): string {
  const words = prompt.toLowerCase().replace(/[^\p{L}\p{N}\s-]/gu, ' ').split(/\s+/).filter((w) => w && !/^(a|an|the|of|and|image|picture|photo|generate|create|make|draw)$/.test(w));
  return words.slice(0, 6).join('-').slice(0, 60) || 'image';
}

const squash = (s: string) => s.toLowerCase().replace(/[\s._-]+/g, '');

/** Models of the service the owner named: by provider, group or a model-family word ("lyria", "veo", "sora"). */
export function byService<T extends { provider: string; group: string; model: string; label: string }>(models: T[], requested: string): T[] {
  const n = squash(requested);
  if (n.length < 3) return [];
  const own = models.filter((m) => squash(m.provider) === n || squash(m.group) === n);
  if (own.length) return own;
  return models.filter((m) => squash(m.model).includes(n) || squash(m.label).includes(n));
}