Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

17.1 KB
Raw Blame
import { afterAll, beforeAll, beforeEach, expect, it as vitestIt, vi } from 'vitest';
import pg from 'pg';
import { mediaDdl } from '../../lib/db/schema/media.js';
import { requireDbOrSkip } from '../../lib/dbTestGate.js';
import { mediaFixture } from '../../../scripts/perf/collectionFixtureData.js';
import { listGalleryCollectionSummaries, listGalleryFacets, listGalleryPage } from './gallery.js';
import { compactGalleryRecord } from './logic.js';
import { ownTestBodies } from '../../lib/mockPathsDataRoot.js';
import { createDbPhaseDiagnostics } from '../../test/dbPhaseDiagnostics.js';

const { it, drain } = ownTestBodies(vitestIt);

const state = vi.hoisted(() => ({ file: false, images: [], videos: [], collections: [], annotations: {}, query: vi.fn() }));
vi.mock('../../lib/runtimeEnv.js', () => ({ isTestRunner: () => state.file }));
vi.mock('../../lib/db.js', () => ({ query: (...args) => state.query(...args) }));
vi.mock('../videoGen/history.js', async () => {
  const { compactSource } = await import('./logic.js');
  return { loadHistory: async () => state.videos, loadCompactHistory: async () => state.videos.map(compactSource) };
});
vi.mock('../mediaCollections.js', () => ({ listCollections: async () => state.collections,
  getCollection: async id => state.collections.find(collection => collection.id === id) }));
vi.mock('../peerHostedMedia.js', () => ({ resolveMediaLocations: async items => items.map(() => ({ location: 'local' })) }));
vi.mock('../mediaAnnotations.js', () => ({ listAnnotations: async () => state.annotations }));
let client;
let observer;
let diagnostics;
let available = false;
beforeAll(async () => {
  vi.stubEnv('MEMORY_BACKEND', 'db');
  if (process.env.PGDATABASE !== 'portos_test') {
    requireDbOrSkip('gallery collection batches', false, 'requires portos_test');
    return;
  }
  const connection = { database: 'portos_test', host: process.env.PGHOST || '127.0.0.1',
    port: Number(process.env.PGPORT || 5432), user: process.env.PGUSER || 'portos',
    password: process.env.PGPASSWORD || 'portos', options: '', connectionTimeoutMillis: 2000 };
  client = new pg.Client(connection);
  await client.connect();
  if (process.env.PORTOS_DB_PHASE_DIAGNOSTICS === '1') {
    observer = new pg.Client({ ...connection, application_name: 'portos-test-diagnostics', statement_timeout: 500, query_timeout: 750 });
    observer.on('error', () => {});
    await observer.connect();
  }
  // A session-private table shadows the index, including for EXPLAIN: no live rows.
  const ddl = mediaDdl.filter(sql => /(?:TABLE IF NOT EXISTS media_assets|INDEX IF NOT EXISTS idx_media_assets_)/.test(sql));
  for (const sql of ddl) await client.query(sql.replace('CREATE TABLE IF NOT EXISTS', 'CREATE TEMP TABLE'));
  // Production uses a pool; this fixture shares one session for its temp table.
  // Serialize that session rather than relying on pg's deprecated query queue.
  let queryTail = Promise.resolve();
  state.query.mockImplementation((...args) => {
    const result = queryTail.then(() => client.query(...args));
    queryTail = result.catch(() => {});
    return result;
  });
  available = true;
});
beforeEach(() => drain());
afterAll(async () => {
  try { await drain().finally(() => diagnostics?.drain()); }
  finally {
    try { await Promise.all([observer?.end(), client?.end()]); }
    finally { vi.unstubAllEnvs(); }
  }
});

async function seed(images, indexedVideos = [], phase = (_name, operation) => operation()) {
  state.images = images;
  await phase('seed-truncate', () => client.query('TRUNCATE pg_temp.media_assets'));
  const rows = await phase('seed-serialize', () => JSON.stringify([...images.map(data => ({ kind: 'image', ref: data.filename, data })),
    ...indexedVideos.map(data => ({ kind: 'video', ref: data.id, data }))]));
  await phase('seed-insert', () => client.query(`INSERT INTO pg_temp.media_assets (media_key, kind, ref, data, created_at)
    SELECT kind || ':' || ref, kind, ref, data, COALESCE((data->>'createdAt')::timestamptz, 'epoch')
    FROM jsonb_to_recordset($1::jsonb) AS r(kind text, ref text, data jsonb)`, [rows]));
}
const member = (kind, ref, addedAt = '2026-01-01') => ({ kind, ref, addedAt });
const disk = async () => state.images;

it('preserves cover fallback, stable membership order, visibility and stale-reference contracts at the public boundary', async context => {
  if (!available) return context.skip();
  await seed([
    { filename: 'a.png', createdAt: '2026-01-01', universeId: 'example', entryKind: 'canon' },
    { filename: 'z.png', path: '/custom/z.png', hidden: true, createdAt: '2026-01-01' },
    { filename: 'loose.png', hidden: true, createdAt: '2026-01-03' },
  ], [{ id: 'stale-index', thumbnail: 'stale.png' }]);
  state.videos = [{ id: 'v', filename: 'v.mp4', thumbnail: 'current.png', hidden: true, createdAt: 'bad-date' },
    { id: 'no-thumb', createdAt: '2026-01-04' }, { id: 'loose', createdAt: '2026-01-05' }];
  const items = [member('image', 'z.png'), member('image', 'a.png'), member('video', 'v'), member('image', 'missing.png')];
  state.collections = [
    { id: 'equal-time', name: 'Equal', items },
    { id: 'pin-image', name: 'Image', items, coverKey: 'image:a.png' },
    { id: 'pin-video', name: 'Video', items, coverKey: 'video:v' },
    { id: 'pin-filename', name: 'Filename', items, coverKey: 'video:v.mp4' },
    { id: 'missing-pin', items, coverKey: 'image:missing.png' },
    { id: 'outside-pin', items, coverKey: 'image:loose.png' },
    { id: 'no-thumbnail', items: [member('video', 'no-thumb'), ...items], coverKey: 'video:no-thumb' },
    { id: 'empty', items: [] }, { id: 'stale', items: [member('image', 'gone.png')] },
    { id: 'hidden-only', items: [member('image', 'z.png')] },
    { id: 'video-only', items: [member('video', 'v')] },
  ];
  state.file = true;
  const expected = await listGalleryCollectionSummaries(disk);
  const facets = await listGalleryFacets(disk);
  expect(expected[0]).toEqual({ id: 'unsorted', cover: '/data/image-thumbnails/loose.webp', total: 2, counts: { image: 1, video: 1, all: 2 } });
  expect(expected[1]).toMatchObject({ cover: '/custom/z.png', total: 4 });
  state.file = false;
  state.query.mockClear();
  expect(await listGalleryCollectionSummaries(disk)).toEqual(expected);
  expect(state.query).toHaveBeenCalledTimes(1);
  state.query.mockClear();
  expect(await listGalleryFacets(disk)).toEqual(facets);
  expect(state.query).toHaveBeenCalledTimes(2);
  // Empty collection list and no authoritative videos still yield Unsorted.
  state.collections = []; state.videos = [];
  expect((await listGalleryCollectionSummaries(disk))[0]).toMatchObject({ total: 3, counts: { image: 3, video: 0, all: 3 } });
  await seed([]);
  state.videos = [{ id: 'loose-video', thumbnail: 'loose.png', hidden: true, createdAt: '2026-01-01' }];
  expect((await listGalleryCollectionSummaries(disk))[0]).toEqual({ id: 'unsorted', total: 1,
    counts: { image: 0, video: 1, all: 1 }, cover: '/data/video-thumbnails/loose.png' });
  state.videos = [];
  expect((await listGalleryCollectionSummaries(disk))[0]).toEqual({ id: 'unsorted', total: 0,
    counts: { image: 0, video: 0, all: 0 }, cover: null });
});

it('keeps summary and picker query counts constant for the synthetic 2400-image/1200-video fixture', async context => {
  if (!available) return context.skip();
  await seed(Array.from({ length: 2400 }, (_, i) => mediaFixture('image', i)));
  state.videos = Array.from({ length: 1200 }, (_, i) => mediaFixture('video', i));
  for (const count of [0, 10, 30]) {
    state.collections = Array.from({ length: count }, (_, i) => ({ id: `collection-${i}`, name: `Collection ${i}`,
      items: state.images.slice(i * 20, i * 20 + 20).map(image => member('image', image.filename)),
    }));
    state.query.mockClear();
    const started = performance.now();
    const summaries = await listGalleryCollectionSummaries(disk);
    const elapsed = performance.now() - started;
    expect(summaries).toHaveLength(count + 1);
    expect(state.query).toHaveBeenCalledTimes(1);
    const [sql, params] = state.query.mock.calls[0];
    const projected = JSON.parse(params[1]);
    expect(projected).toHaveLength(1200);
    expect(projected.every(video => !('prompt' in video) && !('metadata' in video))).toBe(true);
    const plan = (await client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${sql}`, params)).rows[0]['QUERY PLAN'][0].Plan;
    if (process.env.PORTOS_COLLECTION_BENCHMARK === '1') process.stdout.write(JSON.stringify({ collections: count, queries: 1, videoBytes: Buffer.byteLength(params[1]),
      elapsedMs: Math.round(elapsed), responseBytes: Buffer.byteLength(JSON.stringify(summaries)),
      tempReadBlocks: plan['Temp Read Blocks'], tempWrittenBlocks: plan['Temp Written Blocks'] }) + '\n');
    for (let i = 0; i < count; i++) state.collections[i].coverKey = `image:${state.images[i * 20 + 19].filename}`;
    state.query.mockClear();
    const pinned = await listGalleryCollectionSummaries(disk);
    expect(state.query).toHaveBeenCalledTimes(1);
    for (let i = 0; i < count; i++) expect(pinned[i + 1].cover).toBe(state.images[i * 20 + 19].path.replace('/data/images/', '/data/image-thumbnails/').replace(/\.png$/, '.webp'));
    state.query.mockClear();
    expect((await listGalleryFacets(disk)).collections).toHaveLength(count);
    expect(state.query).toHaveBeenCalledTimes(2);
  }
});

// The pre-#9676 contract: full records through SQL, compacted afterwards.
const compactedFull = page => ({ ...page, items: page.items.map(item => item.kind && item.data
  ? { kind: item.kind, data: compactGalleryRecord(item.data) } : compactGalleryRecord(item)) });
const planNodes = plan => [plan, ...(plan.Plans || []).flatMap(planNodes)];

it('projects compact no-search pages before SQL with identical rows, previews and counts', async context => {
  if (!available) return context.skip();
  const beyond = 'needle-beyond-preview';
  const images = [
    { filename: 'nested.png', createdAt: '2026-02-01', prompt: null, metadata: { prompt: 'nested fallback' }, seed: null,
      universeId: 'u1', entryCategory: 'places', entryKind: 'canon', negativePrompt: 'x'.repeat(500) },
    { filename: 'blank.png', createdAt: '2026-02-02', prompt: ' \t\u3000\ufeff', metadata: { prompt: '🦊'.repeat(300) },
      hidden: true, cleanedFrom: 'nested.png', universeId: 'u1' },
    { filename: 'long.png', createdAt: '2026-02-03', prompt: `${'x'.repeat(238)}   ${beyond}`, upscaledFrom: 'nested.png',
      autoCleaned: false, regenerated: true, watermarkRemoved: null },
    { filename: 'short.png', createdAt: '2026-02-04', prompt: 'short ✓ é', width: 0, loraPaths: [], extractedFromVideoId: 'v1' },
    { filename: 'odd.png', createdAt: '2026-02-05', metadata: 'not-an-object', universeId: 'u2', entryCategory: null },
    { filename: 'spaces.png', createdAt: '2026-02-06', prompt: `${'\u00a0'.repeat(300)}late`, stitchedFrom: ['a.png', 'b.png'] },
  ];
  await seed(images, [{ id: 'v1', prompt: 'stale indexed video' }]);
  state.videos = [
    { id: 'v1', filename: 'v1.mp4', thumbnail: 'v1.png', createdAt: '2026-02-07', prompt: `${'é'.repeat(300)} ${beyond}`,
      stitchedFrom: ['a', 'b'], guidanceScale: 3, settings: { steps: 9 } },
    { id: 'v2', filename: 'v2.mp4', thumbnail: null, createdAt: 'invalid', hidden: true, metadata: { prompt: 'video nested' } },
    { id: 'v3', createdAt: '2026-02-08', prompt: `${'\u2028'.repeat(300)}late`, upscaledFrom: 'v1', entryKind: 'canon' },
  ];
  state.collections = [{ id: 'col', name: 'Example', items: [member('video', 'v1', '2026-03-01'),
    member('image', 'short.png', '2026-03-03'), member('image', 'blank.png', '2026-03-02')] }];
  state.annotations = { 'image:long.png': { own: { starred: true } }, 'video:v3': { own: { starred: true } } };
  const cases = [
    {}, { kind: 'image' }, { kind: 'video' }, { hidden: false }, { hidden: true }, { starred: true },
    { starred: true, kind: 'video' }, { collectionId: 'col' }, { collectionId: 'unsorted' }, { filename: 'long.png' },
    { filename: 'v1.mp4', kind: 'video' }, { universeId: 'u1' }, { entryCategory: 'places', entryKind: 'canon' },
    { entryKind: 'canon', kind: 'all' }, { offset: 2, limit: 3 }, { cover: true }, { q: '   ' },
  ];
  const surfaces = [{ media: true, kind: 'all', summary: true }, { media: true, kind: 'all' }, { kind: 'image', summary: true }];
  for (const surface of surfaces) {
    for (const options of cases) {
      const input = { limit: 60, ...surface, ...options };
      const expected = compactedFull(await listGalleryPage({ ...input, compact: false }));
      state.query.mockClear();
      expect(await listGalleryPage({ ...input, compact: true }), JSON.stringify(input)).toEqual(expected);
      for (const [sql, params] of state.query.mock.calls) {
        // Image-only counts read no record payload at all.
        if (!sql.startsWith('SELECT COUNT(*)')) expect(sql).toContain('jsonb_each(data)');
        expect(JSON.stringify(params)).not.toContain(beyond);
      }
    }
  }
  const all = await listGalleryPage({ media: true, kind: 'all', summary: true, compact: true });
  const card = key => all.items.find(item => (item.data.id || item.data.filename) === key).data;
  expect(card('nested.png')).toEqual({ compact: true, filename: 'nested.png', createdAt: '2026-02-01', seed: null,
    prompt: 'nested fallback' });
  expect(card('blank.png').prompt).toBe('🦊'.repeat(120) + '…');
  expect(card('long.png')).toMatchObject({ prompt: 'x'.repeat(238) + '…', watermarkRemoved: null });
  expect(card('spaces.png').prompt).toBe('…');
  expect(card('v1')).toEqual({ compact: true, id: 'v1', filename: 'v1.mp4', thumbnail: 'v1.png', createdAt: '2026-02-07',
    stitchedFrom: ['a', 'b'], prompt: 'é'.repeat(240) + '…' });
  expect(card('v2')).toMatchObject({ thumbnail: null, prompt: 'video nested' });
  expect(card('v3').prompt).toBe('…');

  // Search keeps full-metadata matching beyond the preview, in both kinds.
  const found = await listGalleryPage({ media: true, kind: 'all', summary: true, compact: true, q: beyond });
  expect(found.items.map(item => item.data.id || item.data.filename)).toEqual(['v1', 'long.png']);
  expect(found.items.every(item => item.data.compact)).toBe(true);
  // Non-compact and lookup callers still receive full records with saved prompts/settings.
  const full = await listGalleryPage({ media: true, kind: 'video', filename: 'v1.mp4' });
  expect(full.items).toEqual([{ kind: 'video', data: state.videos[0] }]);
  expect((await listGalleryPage({ limit: 200, mediaKeys: ['image:nested.png'] })).items).toEqual([images[0]]);
});

it('sends no full video detail to SQL and does not spill for the synthetic 2400-image/1200-video page', async context => {
  if (!available) return context.skip();
  diagnostics = createDbPhaseDiagnostics({ onTestFailed: callback => context.onTestFailed(callback), observer, fixturePid: client.processID,
    enabled: process.env.PORTOS_DB_PHASE_DIAGNOSTICS === '1' });
  const { phase } = diagnostics;
  const images = await phase('fixture-images', () => Array.from({ length: 2400 }, (_, i) => mediaFixture('image', i)));
  await seed(images, [], phase);
  state.videos = await phase('fixture-videos', () => Array.from({ length: 1200 }, (_, i) => mediaFixture('video', i)));
  state.collections = [];
  await phase('reset-work-mem', () => client.query('RESET work_mem'));
  const measure = async compact => {
    const label = compact ? 'compact' : 'full';
    state.query.mockClear();
    const page = await phase(`${label}-public-query`, () => listGalleryPage({ media: true, kind: 'all', summary: true, hidden: false, compact, limit: 60 }));
    expect(state.query).toHaveBeenCalledTimes(1);
    const [sql, params] = state.query.mock.calls[0];
    // Measure the actual public-boundary result without executing the same query again.
    const result = await state.query.mock.results[0].value;
    const explained = await phase(`${label}-explain`, () => client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${sql}`, params));
    const nodes = planNodes(explained.rows[0]['QUERY PLAN'][0].Plan);
    return phase(`${label}-result-sizes`, () => ({ page, params, videoBytes: Buffer.byteLength(params[0]), resultBytes: Buffer.byteLength(JSON.stringify(result.rows)),
      responseBytes: Buffer.byteLength(JSON.stringify(page)),
      tempReadBlocks: Math.max(...nodes.map(node => node['Temp Read Blocks'] || 0)),
      tempWrittenBlocks: Math.max(...nodes.map(node => node['Temp Written Blocks'] || 0)),
      executionMs: explained.rows[0]['QUERY PLAN'][0]['Execution Time'] }));
  };
  const before = await measure(false);
  const after = await measure(true);
  expect(after.page).toEqual(compactedFull(before.page));
  expect(after.page.items).toHaveLength(60);
  const videos = JSON.parse(after.params[0]);
  expect(videos).toHaveLength(1200);
  for (const { data } of videos) {
    expect(data).not.toHaveProperty('prompt');
    expect(data._promptPreviewSource.length).toBeLessThanOrEqual(241);
  }
  expect(after.tempReadBlocks).toBe(0);
  expect(after.tempWrittenBlocks).toBe(0);
  expect(after.videoBytes).toBeLessThan(before.videoBytes / 10);
  expect(after.resultBytes).toBeLessThan(before.resultBytes / 10);
  if (process.env.PORTOS_COLLECTION_BENCHMARK === '1') {
    for (const [label, run] of [['full', before], ['compact', after]]) {
      const { page: _page, params: _params, ...metrics } = run;
      process.stdout.write(JSON.stringify({ label, ...metrics }) + '\n');
    }
  }
});