Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

10.7 KB
Raw Blame
/**
 * Calling an image model. Two wire formats cover the providers Mercury has:
 *
 *  - OpenAI Images API: POST /images/generations (JSON) and /images/edits
 *    (multipart, reference images). OpenAI itself, LiteLLM, AI/ML API,
 *    Runcrate, Atlas Cloud, Mercury Cloud, Ollama and most self-hosted
 *    servers expose it. Results come back as b64_json or a URL.
 *  - OpenRouter: chat completions with modalities ["image","text"]; images
 *    come back in choices[0].message.images[].image_url.url as data URLs.
 *    Reference images go in the user message.
 *
 * Options are simple (aspect, quality, count) and mapped to what each
 * format takes; a value a server doesn't take is left out, not guessed.
 */
import { readFileSync } from 'node:fs';
import { basename } from 'node:path';
import type { MediaEndpoint } from '../providers/base.js';
import type { MediaModel } from './catalog.js';
import { mimeOf } from '../artifacts/classify.js';

export type Aspect = 'square' | 'landscape' | 'portrait';
export type Quality = 'auto' | 'low' | 'medium' | 'high';

export interface ImageRequest {
  prompt: string;
  aspect?: Aspect;
  quality?: Quality;
  count?: number;
  /** Local files to edit or take as references. */
  references?: string[];
  signal?: AbortSignal;
}

export interface GeneratedImage {
  bytes: Buffer;
  mime: string;
}

export interface ImageResult {
  images: GeneratedImage[];
  /** The prompt as the model rewrote it, when it says. */
  revisedPrompt?: string;
  /** Text the model wrote alongside (OpenRouter chat). */
  note?: string;
}

const TIMEOUT_MS = 180_000;

export class ImageGenerationError extends Error {
  constructor(message: string, readonly status?: number) {
    super(message);
    this.name = 'ImageGenerationError';
  }
}

function sizeFor(model: string, aspect: Aspect): string {
  if (/dall-e-3/i.test(model)) return aspect === 'landscape' ? '1792x1024' : aspect === 'portrait' ? '1024x1792' : '1024x1024';
  if (/dall-e-2/i.test(model)) return '1024x1024';
  return aspect === 'landscape' ? '1536x1024' : aspect === 'portrait' ? '1024x1536' : '1024x1024';
}

const RATIO: Record<Aspect, string> = { square: '1:1', landscape: '16:9', portrait: '9:16' };

function withTimeout(signal: AbortSignal | undefined): { signal: AbortSignal; done: () => void } {
  const ctrl = new AbortController();
  const timer = setTimeout(() => ctrl.abort(new Error('Image generation timed out')), TIMEOUT_MS);
  const onAbort = () => ctrl.abort(signal?.reason);
  signal?.addEventListener('abort', onAbort, { once: true });
  return { signal: ctrl.signal, done: () => { clearTimeout(timer); signal?.removeEventListener('abort', onAbort); } };
}

async function errorOf(res: Response): Promise<ImageGenerationError> {
  const text = await res.text().catch(() => '');
  let message = text.slice(0, 400);
  try {
    const j = JSON.parse(text) as { error?: { message?: string } | string; message?: string; detail?: string };
    message = (typeof j.error === 'string' ? j.error : j.error?.message) ?? j.message ?? j.detail ?? message;
  } catch { /* not JSON */ }
  return new ImageGenerationError(`${res.status}: ${message || res.statusText}`, res.status);
}

function decodeDataUrl(url: string): GeneratedImage | undefined {
  const m = /^data:([^;,]+)?(;base64)?,(.*)$/s.exec(url);
  if (!m) return undefined;
  return { mime: m[1] || 'image/png', bytes: m[2] ? Buffer.from(m[3], 'base64') : Buffer.from(decodeURIComponent(m[3])) };
}

async function download(url: string, fetchImpl: typeof fetch, signal: AbortSignal): Promise<GeneratedImage> {
  const data = decodeDataUrl(url);
  if (data) return data;
  const res = await fetchImpl(url, { signal });
  if (!res.ok) throw new ImageGenerationError(`Downloading the image failed (${res.status})`, res.status);
  return { mime: res.headers.get('content-type')?.split(';')[0] || 'image/png', bytes: Buffer.from(await res.arrayBuffer()) };
}

function sniffMime(bytes: Buffer, fallback = 'image/png'): string {
  if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return 'image/png';
  if (bytes[0] === 0xff && bytes[1] === 0xd8) return 'image/jpeg';
  if (bytes.subarray(0, 4).toString('ascii') === 'RIFF' && bytes.subarray(8, 12).toString('ascii') === 'WEBP') return 'image/webp';
  if (bytes.subarray(0, 3).toString('ascii') === 'GIF') return 'image/gif';
  return fallback;
}

async function openAIImages(m: MediaModel, ep: MediaEndpoint, req: ImageRequest, fetchImpl: typeof fetch): Promise<ImageResult> {
  const aspect = req.aspect ?? 'square';
  const n = Math.min(Math.max(req.count ?? 1, 1), 4);
  const quality = req.quality && req.quality !== 'auto' && /gpt-image|chatgpt-image/i.test(m.model) ? req.quality : undefined;
  const t = withTimeout(req.signal);
  try {
    let res: Response;
    if (req.references?.length) {
      const form = new FormData();
      form.set('model', m.model);
      form.set('prompt', req.prompt);
      form.set('n', String(n));
      form.set('size', sizeFor(m.model, aspect));
      if (quality) form.set('quality', quality);
      for (const file of req.references.slice(0, 5)) {
        const blob = new Blob([new Uint8Array(readFileSync(file))], { type: mimeOf(file) || 'image/png' });
        form.append(/gpt-image|chatgpt-image/i.test(m.model) ? 'image[]' : 'image', blob, basename(file));
      }
      res = await fetchImpl(`${ep.baseUrl}/images/edits`, { method: 'POST', headers: ep.headers, body: form, signal: t.signal });
    } else {
      res = await fetchImpl(`${ep.baseUrl}/images/generations`, {
        method: 'POST',
        headers: { ...ep.headers, 'Content-Type': 'application/json' },
        body: JSON.stringify({
          model: m.model,
          prompt: req.prompt,
          n,
          size: sizeFor(m.model, aspect),
          ...(quality ? { quality } : {}),
          // dall-e answers with a URL unless asked; gpt-image always sends base64 and rejects the field.
          ...(/dall-e/i.test(m.model) ? { response_format: 'b64_json' } : {}),
        }),
        signal: t.signal,
      });
    }
    if (!res.ok) throw await errorOf(res);
    const body = await res.json() as { data?: Array<{ b64_json?: string; url?: string; revised_prompt?: string }> };
    const images: GeneratedImage[] = [];
    for (const d of body.data ?? []) {
      if (d.b64_json) {
        const bytes = Buffer.from(d.b64_json, 'base64');
        images.push({ bytes, mime: sniffMime(bytes) });
      } else if (d.url) {
        images.push(await download(d.url, fetchImpl, t.signal));
      }
    }
    if (!images.length) throw new ImageGenerationError('The model returned no image');
    const revised = body.data?.find((d) => d.revised_prompt)?.revised_prompt;
    return { images, ...(revised ? { revisedPrompt: revised } : {}) };
  } finally {
    t.done();
  }
}

async function openRouterChat(m: MediaModel, ep: MediaEndpoint, req: ImageRequest, fetchImpl: typeof fetch): Promise<ImageResult> {
  const t = withTimeout(req.signal);
  try {
    const content: Array<Record<string, unknown>> = [{ type: 'text', text: req.prompt }];
    for (const file of req.references?.slice(0, 5) ?? []) {
      content.push({ type: 'image_url', image_url: { url: `data:${mimeOf(file) || 'image/png'};base64,${readFileSync(file).toString('base64')}` } });
    }
    const res = await fetchImpl(`${ep.baseUrl}/chat/completions`, {
      method: 'POST',
      headers: { ...ep.headers, 'Content-Type': 'application/json' },
      body: JSON.stringify({
        model: m.model,
        messages: [{ role: 'user', content }],
        modalities: ['image', 'text'],
        ...(req.aspect ? { image_config: { aspect_ratio: RATIO[req.aspect] } } : {}),
      }),
      signal: t.signal,
    });
    if (!res.ok) throw await errorOf(res);
    const body = await res.json() as { choices?: Array<{ message?: { content?: string; images?: Array<{ image_url?: { url?: string } }> } }> };
    const message = body.choices?.[0]?.message;
    const images: GeneratedImage[] = [];
    for (const img of message?.images ?? []) {
      if (img.image_url?.url) images.push(await download(img.image_url.url, fetchImpl, t.signal));
    }
    if (!images.length) throw new ImageGenerationError(message?.content ? `The model answered in words instead: ${message.content.slice(0, 200)}` : 'The model returned no image');
    return { images, ...(message?.content ? { note: message.content } : {}) };
  } finally {
    t.done();
  }
}

/** Gemini image models (Nano Banana): generateContent with image output; references go in as inline images. */
async function geminiImage(m: MediaModel, ep: MediaEndpoint, req: ImageRequest, fetchImpl: typeof fetch): Promise<ImageResult> {
  const t = withTimeout(req.signal);
  try {
    const parts: Array<Record<string, unknown>> = [{ text: req.prompt }];
    for (const file of req.references?.slice(0, 5) ?? []) {
      parts.push({ inlineData: { mimeType: mimeOf(file) || 'image/png', data: readFileSync(file).toString('base64') } });
    }
    const count = Math.min(Math.max(req.count ?? 1, 1), 4);
    const images: GeneratedImage[] = [];
    let note = '';
    // One image per call: Gemini returns a single picture per candidate.
    for (let i = 0; i < count; i++) {
      const res = await fetchImpl(`${ep.baseUrl}/models/${encodeURIComponent(m.model)}:generateContent`, {
        method: 'POST',
        headers: { ...ep.headers, 'Content-Type': 'application/json' },
        body: JSON.stringify({
          contents: [{ role: 'user', parts }],
          generationConfig: {
            responseModalities: ['TEXT', 'IMAGE'],
            ...(req.aspect ? { imageConfig: { aspectRatio: RATIO[req.aspect] } } : {}),
          },
        }),
        signal: t.signal,
      });
      if (!res.ok) throw await errorOf(res);
      const body = await res.json() as { candidates?: Array<{ content?: { parts?: Array<{ text?: string; inlineData?: { mimeType?: string; data?: string } }> }; finishReason?: string }> };
      for (const p of body.candidates?.[0]?.content?.parts ?? []) {
        if (p.inlineData?.data) images.push({ bytes: Buffer.from(p.inlineData.data, 'base64'), mime: p.inlineData.mimeType || 'image/png' });
        else if (p.text) note += p.text;
      }
      if (!images.length) {
        const reason = body.candidates?.[0]?.finishReason;
        throw new ImageGenerationError(note ? `The model answered in words instead: ${note.slice(0, 200)}` : `The model returned no image${reason ? ` (${reason})` : ''}`);
      }
    }
    return { images, ...(note ? { note } : {}) };
  } finally {
    t.done();
  }
}

export async function generateWith(m: MediaModel, ep: MediaEndpoint, req: ImageRequest, fetchImpl: typeof fetch = globalThis.fetch): Promise<ImageResult> {
  if (m.backend === 'gemini-image') return geminiImage(m, ep, req, fetchImpl);
  return m.backend === 'openrouter-chat' ? openRouterChat(m, ep, req, fetchImpl) : openAIImages(m, ep, req, fetchImpl);
}