Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

20.1 KB
Raw Blame
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { testVision, runVisionTestSuite, checkVisionHealth, describeImageDataUrlDetailed } from './visionTest.js';

// Mock the providers module
vi.mock('./providers.js', () => ({
  getProviderById: vi.fn()
}));

// Mock the CLI vision path so the CLI-dispatch branch is asserted without
// spawning a real child process.
vi.mock('./visionCli.js', () => ({
  describeImageViaCli: vi.fn(),
}));

// Mock the shared managed-runtime wake (issue #8104) so this suite's fixtures
// never depend on the real MTPLX/Slotstream/Ollama manager modules (and the
// heavier deps they in turn statically import, e.g. `sleep` from fileUtils.js).
vi.mock('./providerExecutionReadiness.js', () => ({
  ensureManagedRuntimeReady: vi.fn(() => Promise.resolve({ success: true })),
}));

// Mock fs/promises for image loading and directory listing
vi.mock('fs/promises', () => ({
  readFile: vi.fn(),
  readdir: vi.fn()
}));

// Mock the fileUtils path-safety layer. `resolveScreenshot` is the basename
// allowlist that hardens `imagePath` (issue #1820) — stubbing it lets each test
// drive the resolved-or-rejected outcome without touching the real filesystem.
// Its own traversal/absolute-escape guarantees are covered in fileUtils.test.js.
vi.mock('../lib/fileUtils.js', () => ({
  PATHS: { screenshots: '/mock/screenshots' },
  resolveScreenshot: vi.fn(),
}));

// Import mocked modules
import { getProviderById } from './providers.js';
import { describeImageViaCli } from './visionCli.js';
import { readFile, readdir } from 'fs/promises';
import { resolveScreenshot } from '../lib/fileUtils.js';
import { ensureManagedRuntimeReady } from './providerExecutionReadiness.js';

describe('Vision Test Service', () => {
  const mockProvider = {
    id: 'lmstudio',
    name: 'LM Studio',
    type: 'api',
    endpoint: 'http://localhost:1234/v1',
    apiKey: 'lm-studio',
    models: ['test-vision-model'],
    defaultModel: 'test-vision-model',
    timeout: 60000,
    enabled: true
  };

  const mockImageBuffer = Buffer.from('fake-image-data');

  beforeEach(() => {
    vi.clearAllMocks();
    global.fetch = vi.fn();
  });

  afterEach(() => {
    vi.restoreAllMocks();
  });

  describe('testVision', () => {
    it('should return error when provider not found', async () => {
      getProviderById.mockResolvedValue(null);

      const result = await testVision({
        imagePath: '/test/image.png',
        prompt: 'Describe this image',
        expectedContent: ['test'],
        providerId: 'nonexistent'
      });

      expect(result.success).toBe(false);
      expect(result.error).toContain('not found');
    });

    it('should return error when provider is not API type', async () => {
      getProviderById.mockResolvedValue({
        ...mockProvider,
        type: 'cli'
      });

      const result = await testVision({
        imagePath: '/test/image.png',
        prompt: 'Describe this image',
        expectedContent: ['test'],
        providerId: 'lmstudio'
      });

      expect(result.success).toBe(false);
      expect(result.error).toContain('not an API provider');
    });

    it('should return error when no model specified and no default', async () => {
      getProviderById.mockResolvedValue({
        ...mockProvider,
        defaultModel: null
      });

      const result = await testVision({
        imagePath: '/test/image.png',
        prompt: 'Describe this image',
        expectedContent: ['test']
      });

      expect(result.success).toBe(false);
      expect(result.error).toContain('No model specified');
    });

    it('blocks a keyed provider pointed at a non-allowlisted endpoint and never calls fetch', async () => {
      getProviderById.mockResolvedValue({
        ...mockProvider,
        endpoint: 'https://not-an-allowlisted-host.example',
      });
      resolveScreenshot.mockReturnValue('/mock/screenshots/image.png');
      readFile.mockResolvedValue(mockImageBuffer);

      await expect(testVision({
        imagePath: '/test/image.png',
        prompt: 'Describe this image',
        expectedContent: ['test'],
      })).rejects.toThrow('Blocked outbound API-key request');
      expect(global.fetch).not.toHaveBeenCalled();
    });

    it('should successfully test vision when API returns expected content', async () => {
      getProviderById.mockResolvedValue(mockProvider);
      resolveScreenshot.mockReturnValue('/mock/screenshots/image.png');
      readFile.mockResolvedValue(mockImageBuffer);

      const mockResponse = {
        choices: [{
          message: {
            content: 'This is a screenshot of an application showing a button and text.'
          }
        }],
        usage: { prompt_tokens: 100, completion_tokens: 50 }
      };

      global.fetch.mockResolvedValue({
        ok: true,
        json: () => Promise.resolve(mockResponse)
      });

      const result = await testVision({
        imagePath: '/test/image.png',
        prompt: 'Describe this image',
        expectedContent: ['button', 'text']
      });

      expect(result.success).toBe(true);
      expect(result.model).toBe('test-vision-model');
      expect(result.foundTerms).toContain('button');
      expect(result.foundTerms).toContain('text');
      expect(result.missingTerms).toHaveLength(0);
    });

    it('should return success false when expected content not found', async () => {
      getProviderById.mockResolvedValue(mockProvider);
      resolveScreenshot.mockReturnValue('/mock/screenshots/image.png');
      readFile.mockResolvedValue(mockImageBuffer);

      const mockResponse = {
        choices: [{
          message: {
            content: 'This is a blank image.'
          }
        }]
      };

      global.fetch.mockResolvedValue({
        ok: true,
        json: () => Promise.resolve(mockResponse)
      });

      const result = await testVision({
        imagePath: '/test/image.png',
        prompt: 'Describe this image',
        expectedContent: ['button', 'navigation']
      });

      expect(result.success).toBe(false);
      expect(result.missingTerms).toContain('button');
      expect(result.missingTerms).toContain('navigation');
    });

    it('should handle API errors gracefully', async () => {
      getProviderById.mockResolvedValue(mockProvider);
      resolveScreenshot.mockReturnValue('/mock/screenshots/image.png');
      readFile.mockResolvedValue(mockImageBuffer);

      global.fetch.mockResolvedValue({
        ok: false,
        status: 500,
        text: () => Promise.resolve('Internal Server Error')
      });

      await expect(testVision({
        imagePath: '/test/image.png',
        prompt: 'Describe this image',
        expectedContent: ['test']
      })).rejects.toThrow('Vision API error 500');
    });

    it('should use custom model when specified', async () => {
      getProviderById.mockResolvedValue(mockProvider);
      resolveScreenshot.mockReturnValue('/mock/screenshots/image.png');
      readFile.mockResolvedValue(mockImageBuffer);

      const mockResponse = {
        choices: [{
          message: { content: 'Test response' }
        }]
      };

      global.fetch.mockResolvedValue({
        ok: true,
        json: () => Promise.resolve(mockResponse)
      });

      const result = await testVision({
        imagePath: '/test/image.png',
        prompt: 'Describe',
        expectedContent: [],
        model: 'custom-model'
      });

      expect(result.model).toBe('custom-model');

      // Verify the API was called with custom model
      const fetchCall = global.fetch.mock.calls[0];
      const body = JSON.parse(fetchCall[1].body);
      expect(body.model).toBe('custom-model');
    });

    it('should handle image not found error', async () => {
      getProviderById.mockResolvedValue(mockProvider);
      resolveScreenshot.mockReturnValue(null);

      await expect(testVision({
        imagePath: '/nonexistent/image.png',
        prompt: 'Describe',
        expectedContent: []
      })).rejects.toThrow('Failed to load image');
    });

    it('SECURITY: refuses a traversal/absolute imagePath without reading the file or calling the provider (#1820)', async () => {
      getProviderById.mockResolvedValue(mockProvider);
      // resolveScreenshot rejects anything that escapes the screenshots root.
      resolveScreenshot.mockReturnValue(null);

      await expect(testVision({
        imagePath: '../../../../etc/passwd',
        prompt: 'Describe',
        expectedContent: []
      })).rejects.toThrow('Failed to load image');

      // The unsanitized path must never reach readFile or the external API —
      // otherwise arbitrary file contents would be base64-encoded and forwarded.
      expect(resolveScreenshot).toHaveBeenCalledWith('../../../../etc/passwd');
      expect(readFile).not.toHaveBeenCalled();
      expect(global.fetch).not.toHaveBeenCalled();
    });
  });

  describe('runVisionTestSuite', () => {
    it('should return error when no screenshots available', async () => {
      readdir.mockResolvedValue([]);

      const result = await runVisionTestSuite('lmstudio');

      expect(result.success).toBe(false);
      expect(result.error).toContain('No screenshots available');
    });

    it('should run multiple tests on available screenshots', async () => {
      readdir.mockResolvedValue(['test1.png', 'test2.jpg']);
      getProviderById.mockResolvedValue(mockProvider);
      resolveScreenshot.mockReturnValue('/mock/screenshots/image.png');
      readFile.mockResolvedValue(mockImageBuffer);

      const mockResponse = {
        choices: [{
          message: { content: 'This is a detailed description of what I see in the image.' }
        }]
      };

      global.fetch.mockResolvedValue({
        ok: true,
        json: () => Promise.resolve(mockResponse)
      });

      const result = await runVisionTestSuite('lmstudio');

      expect(result.totalTests).toBe(2);
      expect(result.results).toHaveLength(2);
      expect(result.results[0].testName).toBe('basic-description');
      expect(result.results[1].testName).toBe('ui-identification');
    });
  });

  describe('describeImageDataUrlDetailed', () => {
    const DATA_URL = 'data:image/png;base64,Zm9v';
    const callWith = async (message, extra = {}) => {
      getProviderById.mockResolvedValue(mockProvider);
      global.fetch.mockResolvedValue({
        ok: true,
        json: () => Promise.resolve({ choices: [{ message, ...extra }], usage: extra.usage }),
      });
      return describeImageDataUrlDetailed({ dataUrl: DATA_URL, prompt: 'caption this' });
    };

    it('surfaces finish_reason and usage alongside the text', async () => {
      const out = await callWith(
        { content: 'a calm portrait, soft light' },
        { finish_reason: 'stop', usage: { completion_tokens: 12 } },
      );
      expect(out.text).toBe('a calm portrait, soft light');
      expect(out.finishReason).toBe('stop');
      expect(out.usage).toEqual({ completion_tokens: 12 });
      expect(out.reasoning).toBe('');
    });

    it('maps a length cutoff to finishReason for the empty-caption diagnosis', async () => {
      const out = await callWith({ content: '' }, { finish_reason: 'length', usage: { completion_tokens: 600 } });
      expect(out.text).toBe('');
      expect(out.finishReason).toBe('length');
      expect(out.usage.completion_tokens).toBe(600);
    });

    it('extracts reasoning from each backend field shape (Ollama / LM Studio / native)', async () => {
      expect((await callWith({ content: '', reasoning: 'ollama thoughts' })).reasoning).toBe('ollama thoughts');
      expect((await callWith({ content: '', reasoning_content: 'lmstudio thoughts' })).reasoning).toBe('lmstudio thoughts');
      expect((await callWith({ content: '', thinking: 'native thoughts' })).reasoning).toBe('native thoughts');
    });

    it('pulls reasoning out of an inline <think> block and strips it from the caption text', async () => {
      const out = await callWith({ content: '<think>she is facing left</think>bust shot, looking left' });
      expect(out.reasoning).toBe('she is facing left');
      // the leaked reasoning must NOT pollute the persisted caption
      expect(out.text).toBe('bust shot, looking left');
    });

    it('returns empty text (not a stray tag) when the reply is only a think block', async () => {
      const out = await callWith({ content: '<think>I should refuse</think>' }, { finish_reason: 'stop' });
      expect(out.text).toBe('');
      expect(out.reasoning).toBe('I should refuse');
    });

    it('delegates a CLI provider to the CLI vision path (not the API call)', async () => {
      getProviderById.mockResolvedValue({ id: 'codex', type: 'cli', command: 'codex', defaultModel: 'gpt-5' });
      describeImageViaCli.mockResolvedValue({ text: 'a portrait', finishReason: null, usage: null, reasoning: '' });
      const out = await describeImageDataUrlDetailed({ dataUrl: DATA_URL, prompt: 'caption', providerId: 'codex' });
      expect(out.text).toBe('a portrait');
      expect(describeImageViaCli).toHaveBeenCalledWith(expect.objectContaining({
        provider: expect.objectContaining({ id: 'codex' }), dataUrl: DATA_URL, prompt: 'caption',
      }));
      expect(global.fetch).not.toHaveBeenCalled();
    });

    it('still rejects a non-api / non-cli provider (e.g. tui)', async () => {
      getProviderById.mockResolvedValue({ id: 'codex-tui', type: 'tui', command: 'codex' });
      await expect(describeImageDataUrlDetailed({ dataUrl: DATA_URL, prompt: 'x', providerId: 'codex-tui' }))
        .rejects.toThrow(/not an API provider/);
    });

    it('forwards the CLI provider timeout to the CLI vision path', async () => {
      getProviderById.mockResolvedValue({ id: 'codex', type: 'cli', command: 'codex', timeout: 300000 });
      describeImageViaCli.mockResolvedValue({ text: 'ok', finishReason: null, usage: null, reasoning: '' });
      await describeImageDataUrlDetailed({ dataUrl: DATA_URL, prompt: 'caption', providerId: 'codex' });
      expect(describeImageViaCli).toHaveBeenCalledWith(expect.objectContaining({ timeout: 300000 }));
    });

    // Issue #8104: before this fix, the vision path only recognized Ollama on
    // its DEFAULT port (it called `ensureOllamaProviderReady({ endpoint })`,
    // dropping the provider's id/name) and never woke MTPLX or Slotstream at
    // all. `callVisionAPI` now passes the full provider record through the
    // shared `ensureManagedRuntimeReady`, so recognition (by id/name, not just
    // the default port) and MTPLX/Slotstream coverage are both the shared
    // gate's job, not a vision-specific copy.
    describe('managed-runtime wake', () => {
      it('wakes an MTPLX-recognized API provider before calling it', async () => {
        const mtplxProvider = {
          id: 'mtplx', name: 'MTPLX', type: 'api', endpoint: 'http://127.0.0.1:8000/v1', defaultModel: 'mtplx-vision',
        };
        getProviderById.mockResolvedValue(mtplxProvider);
        global.fetch.mockResolvedValue({
          ok: true,
          json: () => Promise.resolve({ choices: [{ message: { content: 'a scene' } }] }),
        });

        await describeImageDataUrlDetailed({ dataUrl: DATA_URL, prompt: 'caption', providerId: 'mtplx' });

        expect(ensureManagedRuntimeReady).toHaveBeenCalledWith(expect.objectContaining({ id: 'mtplx' }));
      });

      // The provider carries its id/name alongside a non-default port — the
      // shared gate (not vision-specific string matching on `:11434`) is what
      // makes this recognizable as Ollama.
      it('passes the full provider record (not just { endpoint }) so an Ollama provider on a non-default port is still recognized', async () => {
        const ollamaProvider = {
          id: 'ollama', name: 'Ollama', type: 'api', endpoint: 'http://127.0.0.1:41434/v1', defaultModel: 'llava',
        };
        getProviderById.mockResolvedValue(ollamaProvider);
        global.fetch.mockResolvedValue({
          ok: true,
          json: () => Promise.resolve({ choices: [{ message: { content: 'a scene' } }] }),
        });

        await describeImageDataUrlDetailed({ dataUrl: DATA_URL, prompt: 'caption', providerId: 'ollama' });

        expect(ensureManagedRuntimeReady).toHaveBeenCalledWith(
          expect.objectContaining({ id: 'ollama', endpoint: 'http://127.0.0.1:41434/v1' }),
        );
      });

      it('never calls the vision endpoint when the managed runtime fails to wake', async () => {
        getProviderById.mockResolvedValue({
          id: 'mtplx', type: 'api', endpoint: 'http://127.0.0.1:8000/v1', defaultModel: 'mtplx-vision',
        });
        ensureManagedRuntimeReady.mockResolvedValueOnce({
          success: false,
          error: 'MTPLX is not running and PortOS could not start it: checkpoint failed to load',
        });

        await expect(describeImageDataUrlDetailed({ dataUrl: DATA_URL, prompt: 'caption', providerId: 'mtplx' }))
          .rejects.toThrow('MTPLX is not running and PortOS could not start it: checkpoint failed to load');
        expect(global.fetch).not.toHaveBeenCalled();
      });
    });
  });

  describe('checkVisionHealth', () => {
    it('should return unavailable when provider not found', async () => {
      getProviderById.mockResolvedValue(null);

      const result = await checkVisionHealth('nonexistent');

      expect(result.available).toBe(false);
      expect(result.error).toContain('not found');
    });

    it('should return unavailable when provider is disabled', async () => {
      getProviderById.mockResolvedValue({
        ...mockProvider,
        enabled: false
      });

      const result = await checkVisionHealth('lmstudio');

      expect(result.available).toBe(false);
      expect(result.error).toContain('disabled');
    });

    it('should return unavailable when provider is not API type', async () => {
      getProviderById.mockResolvedValue({
        ...mockProvider,
        type: 'cli'
      });

      const result = await checkVisionHealth('lmstudio');

      expect(result.available).toBe(false);
      expect(result.error).toContain('requires API provider');
    });

    it('should return available when endpoint is reachable', async () => {
      getProviderById.mockResolvedValue(mockProvider);

      global.fetch.mockResolvedValue({
        ok: true,
        json: () => Promise.resolve({ data: [] })
      });

      const result = await checkVisionHealth('lmstudio');

      expect(result.available).toBe(true);
      expect(result.provider).toBe('lmstudio');
      expect(result.endpoint).toBe(mockProvider.endpoint);
    });

    it('should return unavailable when endpoint not reachable', async () => {
      getProviderById.mockResolvedValue(mockProvider);

      global.fetch.mockResolvedValue({
        ok: false
      });

      const result = await checkVisionHealth('lmstudio');

      expect(result.available).toBe(false);
      expect(result.error).toContain('not reachable');
    });

    it('blocks a keyed provider pointed at a cloud-metadata endpoint and never calls fetch', async () => {
      getProviderById.mockResolvedValue({
        ...mockProvider,
        endpoint: 'http://169.254.169.254/latest/meta-data/',
      });

      const result = await checkVisionHealth('lmstudio');

      expect(result.available).toBe(false);
      expect(result.error).toContain('Provider endpoint blocked');
      expect(global.fetch).not.toHaveBeenCalled();
    });

    it('blocks a keyed provider on a non-allowlisted host when allowCustomEndpoint is not set', async () => {
      getProviderById.mockResolvedValue({
        ...mockProvider,
        endpoint: 'https://not-an-allowlisted-host.example',
      });

      const result = await checkVisionHealth('lmstudio');

      expect(result.available).toBe(false);
      expect(result.error).toContain('Provider endpoint blocked');
      expect(global.fetch).not.toHaveBeenCalled();
    });

    it('allows a keyed provider on a non-allowlisted host through when allowCustomEndpoint is true', async () => {
      getProviderById.mockResolvedValue({
        ...mockProvider,
        endpoint: 'https://not-an-allowlisted-host.example',
        allowCustomEndpoint: true,
      });
      global.fetch.mockResolvedValue({ ok: true, json: () => Promise.resolve({ data: [] }) });

      const result = await checkVisionHealth('lmstudio');

      expect(result.available).toBe(true);
      expect(global.fetch).toHaveBeenCalled();
    });
  });
});