Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

17.1 KB
Raw Blame
/**
 * Vision Test Service
 *
 * Tests LM Studio's vision capabilities by sending images to the API
 * and verifying the model can correctly interpret them.
 */

import { readFile, readdir } from 'fs/promises';
import { extname } from 'path';
import { getProviderById } from './providers.js';
import { PATHS, resolveScreenshot } from '../lib/fileUtils.js';
import { describeFrameStats, isDegenerateFrame } from '../lib/imageFrameStats.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
import { describeImageViaCli } from './visionCli.js';
import { assertSecretEndpoint, evaluateSecretEndpoint } from '../lib/aiToolkit/endpointGuard.js';

const SCREENSHOTS_DIR = PATHS.screenshots;
const DEFAULT_VISION_TIMEOUT_MS = 60000;
const VISION_HEALTH_TIMEOUT_MS = 5000;

/**
 * Get MIME type from file extension
 * @param {string} filepath - Path to the file
 * @returns {string} - MIME type
 */
function getMimeType(filepath) {
  const ext = extname(filepath).toLowerCase();
  const mimeTypes = {
    '.png': 'image/png',
    '.jpg': 'image/jpeg',
    '.jpeg': 'image/jpeg',
    '.gif': 'image/gif',
    '.webp': 'image/webp'
  };
  return mimeTypes[ext] || 'image/png';
}

/**
 * Load an image as base64 data URL
 * @param {string} imagePath - Path to the image file
 * @returns {Promise<string>} - Base64 data URL
 */
async function loadImageAsBase64(imagePath) {
  // `imagePath` originates from req.body — basename-allowlist it to a real file
  // under SCREENSHOTS_DIR. This rejects `../` traversal and absolute-path
  // escapes so an attacker can't have an arbitrary file base64-encoded and
  // forwarded to the external vision provider (issue #1820).
  const fullPath = resolveScreenshot(imagePath);
  if (!fullPath) {
    throw new Error(`Image not found under screenshots dir: ${imagePath}`);
  }

  const buffer = await readFile(fullPath);
  // Degenerate-frame gate (#4173): never spend a vision-provider call asking a
  // model to describe a solid black square. Only a MEASURED degenerate verdict
  // refuses — an image whose stats could not be computed (`ok: null`) is sent
  // as before, because "couldn't measure" is not "has no content".
  const stats = await describeFrameStats(buffer);
  if (isDegenerateFrame(stats)) {
    throw new Error(`image has no content (${stats.reason}) — not sending it to the vision provider`);
  }
  const mimeType = getMimeType(fullPath);
  return `data:${mimeType};base64,${buffer.toString('base64')}`;
}

/**
 * Call LM Studio API with vision request
 * @param {Object} options - Request options
 * @param {Object} [options.provider] - The full provider record, so a managed
 *   local runtime (Ollama, MTPLX, Slotstream) on a non-default endpoint is
 *   still recognized by id/name, not just its default port. Falls back to
 *   `{ endpoint }` when omitted, matching the recognition PortOS could do
 *   before this parameter existed.
 * @param {string} options.endpoint - API endpoint
 * @param {string} options.apiKey - API key
 * @param {string} options.model - Model to use
 * @param {string} options.imageDataUrl - Base64 image data URL
 * @param {string} options.prompt - Prompt to send with image
 * @param {number} options.timeout - Request timeout in ms
 * @param {number} [options.maxTokens=500] - Completion budget. The default suits
 *   short "describe this" calls; callers that ask the model for a larger
 *   structured response (e.g. a JSON object) must raise it or the reply
 *   truncates mid-output and fails to parse.
 * @returns {Promise<Object>} - API response
 */
async function callVisionAPI({ provider, endpoint, apiKey, allowCustomEndpoint, model, imageDataUrl, prompt, timeout = DEFAULT_VISION_TIMEOUT_MS, maxTokens = 500 }) {
  // Never send the API key to an arbitrary/metadata host (SSRF / key
  // exfiltration). Keyless local-LLM calls skip this guard entirely.
  assertSecretEndpoint(endpoint, { hasSecret: Boolean(apiKey), allowCustomEndpoint: allowCustomEndpoint === true });

  // Deferred so visionTest.js's static import closure stays flat — it has
  // many importers, and importScoping.test.js enforces the budget.
  // See root AGENTS.md's #8104 pattern: every inference entry point routes
  // through this one managed-runtime wake instead of hand-rolling Ollama-only
  // dispatch.
  const { ensureManagedRuntimeReady } = await import('./providerExecutionReadiness.js');
  const ready = await ensureManagedRuntimeReady(provider || { endpoint });
  if (!ready.success) throw new Error(ready.error);

  const response = await fetchWithTimeout(`${endpoint}/chat/completions`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': `Bearer ${apiKey}`
    },
    body: JSON.stringify({
      model,
      messages: [
        {
          role: 'user',
          content: [
            {
              type: 'image_url',
              image_url: {
                url: imageDataUrl
              }
            },
            {
              type: 'text',
              text: prompt
            }
          ]
        }
      ],
      max_tokens: maxTokens,
      temperature: 0.1
    })
  }, timeout);

  if (!response.ok) {
    const errorText = await response.text().catch(() => 'Unknown error');
    throw new Error(`Vision API error ${response.status}: ${errorText}`);
  }

  return response.json();
}

/**
 * Test vision capability with a specific image and prompt
 * @param {Object} options - Test options
 * @param {string} options.imagePath - Path to test image
 * @param {string} options.prompt - Test prompt
 * @param {string} options.expectedContent - Keywords expected in response
 * @param {string} [options.providerId='lmstudio'] - Provider to use
 * @param {string} [options.model] - Model to use (defaults to provider's default)
 * @returns {Promise<Object>} - Test result
 */
export async function testVision({ imagePath, prompt, expectedContent, providerId = 'lmstudio', model }) {
  const startTime = Date.now();

  // Get provider configuration. The AI toolkit warms at server boot, so an
  // AI_TOOLKIT_NOT_INITIALIZED code only fires from a vision test that races
  // boot — surface the boot state directly rather than masquerading as "not
  // found", which would send the user looking for the wrong root cause.
  let provider;
  try {
    provider = await getProviderById(providerId);
  } catch (err) {
    return {
      success: false,
      error: err.code === 'AI_TOOLKIT_NOT_INITIALIZED'
        ? 'AI provider service is still initializing — try again in a moment'
        : err.message,
      duration: Date.now() - startTime
    };
  }
  if (!provider) {
    return {
      success: false,
      error: `Provider '${providerId}' not found`,
      duration: Date.now() - startTime
    };
  }

  if (provider.type !== 'api') {
    return {
      success: false,
      error: `Provider '${providerId}' is not an API provider (type: ${provider.type})`,
      duration: Date.now() - startTime
    };
  }

  const testModel = model || provider.defaultModel;
  if (!testModel) {
    return {
      success: false,
      error: 'No model specified and provider has no default model',
      duration: Date.now() - startTime
    };
  }

  // Load image
  const imageDataUrl = await loadImageAsBase64(imagePath).catch(err => {
    throw new Error(`Failed to load image: ${err.message}`);
  });

  console.log(`🔍 Testing vision with model: ${testModel} | image: ${imagePath}`);

  // Call vision API
  const apiResponse = await callVisionAPI({
    provider,
    endpoint: provider.endpoint,
    apiKey: provider.apiKey,
    allowCustomEndpoint: provider.allowCustomEndpoint,
    model: testModel,
    imageDataUrl,
    prompt,
    timeout: provider.timeout || DEFAULT_VISION_TIMEOUT_MS
  });

  const responseContent = apiResponse.choices?.[0]?.message?.content || '';
  const duration = Date.now() - startTime;

  // Check if expected content is present
  const expectedTerms = Array.isArray(expectedContent) ? expectedContent : [expectedContent];
  const foundTerms = expectedTerms.filter(term =>
    responseContent.toLowerCase().includes(term.toLowerCase())
  );
  const allFound = foundTerms.length === expectedTerms.length;

  console.log(`✅ Vision test completed in ${duration}ms`);
  console.log(`📝 Response: ${responseContent.substring(0, 200)}...`);

  return {
    success: allFound,
    model: testModel,
    provider: providerId,
    imagePath,
    prompt,
    response: responseContent,
    expectedTerms,
    foundTerms,
    missingTerms: expectedTerms.filter(t => !foundTerms.includes(t)),
    duration,
    usage: apiResponse.usage || null
  };
}

/**
 * Describe an in-memory image (base64 data URL) using a provider's vision
 * endpoint and return the model's text. Used by the voice agent's
 * ui_describe_visually tool, which captures a screenshot client-side rather
 * than reading one off disk. Throws on provider/transport errors so the caller
 * can surface them.
 * @param {Object} options
 * @param {string} options.dataUrl - base64 image data URL (data:image/...;base64,...)
 * @param {string} options.prompt - What to ask about the image
 * @param {string} [options.providerId='lmstudio'] - Vision-capable API provider
 * @param {string} [options.model] - Model override (defaults to provider default)
 * @param {number} [options.maxTokens] - Completion budget; raise it above the
 *   default when asking the model for a long/structured reply (see callVisionAPI).
 * @returns {Promise<string>} - The model's description text
 */
export async function describeImageDataUrl(opts) {
  return (await describeImageDataUrlDetailed(opts)).text;
}

// Pull any hidden chain-of-thought a reasoning model emitted out of band, so an
// empty `content` can be explained ("it reasoned but produced no caption")
// rather than mislabeled as a refusal. Backends surface this differently:
// Ollama's OpenAI-compat puts it on `message.reasoning`, LM Studio on
// `message.reasoning_content`, and some models leak an inline `<think>…</think>`
// block into `content`. Returns the reasoning text (trimmed) or ''.
function extractReasoning(message) {
  if (!message || typeof message !== 'object') return '';
  const direct = message.reasoning || message.reasoning_content || message.thinking;
  if (typeof direct === 'string' && direct.trim()) return direct.trim();
  const content = typeof message.content === 'string' ? message.content : '';
  const inline = content.match(/<think>([\s\S]*?)<\/think>/i);
  return inline ? inline[1].trim() : '';
}

/**
 * Like {@link describeImageDataUrl} but returns the model's text alongside the
 * diagnostics that explain a blank reply — `finishReason` ('length' = cut off
 * at the token budget), token `usage`, and any out-of-band `reasoning`. The
 * caption loop uses these to tell a real refusal apart from a reasoning model
 * that burned its whole budget thinking and returned no caption.
 * @returns {Promise<{ text: string, finishReason: string|null, usage: object|null, reasoning: string }>}
 */
export async function describeImageDataUrlDetailed({ dataUrl, prompt, providerId = 'lmstudio', model, maxTokens }) {
  if (typeof dataUrl !== 'string' || !dataUrl.startsWith('data:image/')) {
    throw new Error('dataUrl must be a base64 image data URL');
  }
  const provider = await getProviderById(providerId);
  if (!provider) throw new Error(`Provider '${providerId}' not found`);
  // CLI providers (codex / claude-code with vision) read the image from a file
  // rather than an `image_url` block — delegate to the CLI vision path, which
  // returns the same { text, finishReason, usage, reasoning } shape. maxTokens
  // has no CLI analogue (the CLI manages its own budget) so it's dropped.
  if (provider.type === 'cli') {
    return describeImageViaCli({
      provider, dataUrl, prompt: prompt || 'Describe what you see in this image.', model,
      // Honor the provider's configured timeout (seeded codex/claude CLI
      // providers allow 300s/900s) — a vision caption/crop run can outlast the
      // CLI path's 120s default. Falls back to that default when unset.
      ...(provider.timeout ? { timeout: provider.timeout } : {}),
    });
  }
  if (provider.type !== 'api') throw new Error(`Provider '${providerId}' is not an API provider (type: ${provider.type})`);
  const visionModel = model || provider.defaultModel;
  if (!visionModel) throw new Error('No model specified and provider has no default model');

  const apiResponse = await callVisionAPI({
    provider,
    endpoint: provider.endpoint,
    apiKey: provider.apiKey,
    allowCustomEndpoint: provider.allowCustomEndpoint,
    model: visionModel,
    imageDataUrl: dataUrl,
    prompt: prompt || 'Describe what you see in this image.',
    timeout: provider.timeout || DEFAULT_VISION_TIMEOUT_MS,
    ...(maxTokens != null ? { maxTokens } : {}),
  });
  const choice = apiResponse.choices?.[0];
  // A reasoning model that leaks its chain-of-thought inline emits
  // `<think>…</think>` followed by the real answer; strip a LEADING think block
  // so the returned text is the caption (or JSON, for the digital-twin caller),
  // not the caption with the model's reasoning glued to the front. Anchored to
  // the start so legitimate mid-text never gets clipped; the reasoning itself is
  // still surfaced via `reasoning` for the diagnostic.
  const rawContent = choice?.message?.content || '';
  return {
    text: rawContent.replace(/^\s*<think>[\s\S]*?<\/think>\s*/i, '').trim(),
    finishReason: choice?.finish_reason || null,
    usage: apiResponse.usage || null,
    reasoning: extractReasoning(choice?.message),
  };
}

/**
 * Run a comprehensive vision test suite
 * @param {string} [providerId='lmstudio'] - Provider to test
 * @param {string} [model] - Specific model to test
 * @returns {Promise<Object>} - Test suite results
 */
export async function runVisionTestSuite(providerId = 'lmstudio', model) {
  const results = [];
  const allFiles = await readdir(SCREENSHOTS_DIR).catch(() => []);
  const screenshotFiles = allFiles.filter(f =>
    /\.(png|jpg|jpeg|gif|webp)$/i.test(f)
  );

  if (screenshotFiles.length === 0) {
    return {
      success: false,
      error: 'No screenshots available for testing',
      results: []
    };
  }

  // Use first available screenshot for basic vision test
  const testImage = screenshotFiles[0];

  // Test 1: Basic image description
  const describeTest = await testVision({
    imagePath: testImage,
    prompt: 'Describe what you see in this image in 2-3 sentences. Focus on the main elements visible.',
    expectedContent: [], // No specific content expected, just verify it responds
    providerId,
    model
  }).catch(err => ({
    success: false,
    error: err.message,
    testName: 'basic-description'
  }));

  results.push({ ...describeTest, testName: 'basic-description' });

  // Test 2: UI element identification (if it's an app screenshot)
  const uiTest = await testVision({
    imagePath: testImage,
    prompt: 'If this is a screenshot of an application or website, identify any visible UI elements like buttons, forms, navigation, or text. If not a UI screenshot, describe the main subject.',
    expectedContent: [], // Just verify response quality
    providerId,
    model
  }).catch(err => ({
    success: false,
    error: err.message,
    testName: 'ui-identification'
  }));

  results.push({ ...uiTest, testName: 'ui-identification' });

  // Calculate overall success
  const successfulTests = results.filter(r => !r.error && r.response?.length > 20);

  return {
    success: successfulTests.length === results.length,
    totalTests: results.length,
    passedTests: successfulTests.length,
    provider: providerId,
    model: model || 'default',
    results
  };
}

/**
 * Quick health check for vision capabilities
 * @param {string} [providerId='lmstudio'] - Provider to check
 * @returns {Promise<Object>} - Health check result
 */
export async function checkVisionHealth(providerId = 'lmstudio') {
  let provider;
  try {
    provider = await getProviderById(providerId);
  } catch (err) {
    return { available: false, error: err.message };
  }

  if (!provider) {
    return { available: false, error: 'Provider not found' };
  }

  if (!provider.enabled) {
    return { available: false, error: 'Provider is disabled' };
  }

  if (provider.type !== 'api') {
    return { available: false, error: 'Vision requires API provider' };
  }

  // Never send the API key to an arbitrary/metadata host (SSRF / key
  // exfiltration). Keyless local-LLM calls skip this guard entirely. This
  // function's convention is to return { available: false, error } rather
  // than throw, so evaluate (not assert) the guard.
  if (provider.apiKey) {
    const guard = evaluateSecretEndpoint(provider.endpoint, {
      allowCustomEndpoint: provider.allowCustomEndpoint === true,
    });
    if (!guard.allowed) {
      return { available: false, error: `Provider endpoint blocked: ${guard.reason}` };
    }
  }

  // Check if endpoint is reachable
  const response = await fetchWithTimeout(`${provider.endpoint}/models`, {
    headers: provider.apiKey ? { 'Authorization': `Bearer ${provider.apiKey}` } : {}
  }, VISION_HEALTH_TIMEOUT_MS).catch(() => null);

  if (!response?.ok) {
    return { available: false, error: 'API endpoint not reachable' };
  }

  return {
    available: true,
    provider: providerId,
    endpoint: provider.endpoint,
    defaultModel: provider.defaultModel
  };
}