Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

7.8 KB
Raw Blame
/**
 * Tests for the legacy data/universes/{id}/index.json → Postgres importer (#1014).
 *
 * The legacy dir walk (readdir/stat), per-record JSON reads (readJSONFile),
 * marker I/O (readFile/writeFile), and the dir rename are mocked to in-memory
 * state; `db.query` is a fake honoring ON CONFLICT DO NOTHING. No disk, no PG.
 */

import { describe, it, expect, beforeEach, vi } from 'vitest';
import { posixPath as toPosix } from '../lib/testHelper.js';

// In-memory marker/legacy state.
let files = {};            // path → string (markers)
let dirEntries = null;     // readdir(legacyDir) result, or null = ENOENT
let recordsByDir = {};     // `${legacyDir}/${id}/index.json` → record object
let typeIndex = null;      // legacyDir/index.json content
let renameShouldFail = false;
let renamed = [];

vi.mock('../lib/fileUtils.js', () => ({
  PATHS: { data: '/fake/data' },
  readJSONFile: vi.fn(async (path) => {
    if (toPosix(path) === '/fake/data/universes/index.json') return typeIndex;
    return recordsByDir[toPosix(path)] ?? null;
  }),
}));

// Marker I/O is delegated to migrationMarker.js; mock it against the same
// in-memory `files` map so the walk's gate/stamp behavior is exercised without
// the marker helper's real fs (the helper has its own unit test).
vi.mock('../lib/migrationMarker.js', () => ({
  markerExists: vi.fn(async (name) => (`/fake/data/${name}` in files)),
  writeMarker: vi.fn(async (name, payload) => { files[`/fake/data/${name}`] = JSON.stringify(payload, null, 2); }),
}));

vi.mock('fs/promises', () => ({
  readFile: vi.fn(async (path) => {
    const key = toPosix(path);
    if (key === '/fake/data/universes/index.json' && typeIndex !== null) return JSON.stringify(typeIndex);
    if (!(key in recordsByDir)) { const error = new Error('ENOENT'); error.code = 'ENOENT'; throw error; }
    return JSON.stringify(recordsByDir[key]);
  }),
  rename: vi.fn(async (from, to) => {
    if (renameShouldFail) throw new Error('EACCES');
    renamed.push([toPosix(from), toPosix(to)]);
  }),
  readdir: vi.fn(async () => {
    if (dirEntries === null) return [];
    return dirEntries.map(name => ({ name, isDirectory: () => name !== 'index.json' && !name.startsWith('.') }));
  }),
  stat: vi.fn(async (path) => {
    if (toPosix(path) === '/fake/data/universes') {
      if (dirEntries === null) { const e = new Error('ENOENT'); e.code = 'ENOENT'; throw e; }
      return { isDirectory: () => true };
    }
    const e = new Error('ENOENT'); e.code = 'ENOENT'; throw e;
  }),
}));

let uTable;
let rTable;
// jsonb refuses the ASCII escape of an unpaired surrogate (`\ud83d`), as PostgreSQL does (#12815).
const assertJsonbAccepts = (text) => {
  const unpaired = text.replace(/\\ud[89ab][0-9a-f]{2}\\ud[c-f][0-9a-f]{2}/gi, '').match(/\\ud[89a-f][0-9a-f]{2}/i);
  if (unpaired) throw new Error(`invalid input syntax for type json: lone surrogate ${unpaired[0]}`);
};
const query = vi.fn(async (sql, params) => {
  if (/INSERT INTO universes/.test(sql)) {
    assertJsonbAccepts(params[2]);
    const id = params[0];
    if (uTable.has(id)) return { rowCount: 0, rows: [] };
    uTable.set(id, { id, name: params[1], data: JSON.parse(params[2]) });
    return { rowCount: 1, rows: [] };
  }
  if (/INSERT INTO universe_runs/.test(sql)) {
    assertJsonbAccepts(params[3]);
    const id = params[0];
    if (rTable.has(id)) return { rowCount: 0, rows: [] };
    rTable.set(id, { id, universeId: params[1] });
    return { rowCount: 1, rows: [] };
  }
  return { rowCount: 0, rows: [] };
});
vi.mock('../lib/db.js', () => ({ query: (...a) => query(...a) }));

const MARKER = '/fake/data/universes.migrated.json';
const rec = (id, extra = {}) => ({ id, name: id, schemaVersion: 4, createdAt: '2026-01-01T00:00:00.000Z', updatedAt: '2026-01-02T00:00:00.000Z', ...extra });

async function importer() {
  const { migrateUniversesToDB } = await import('./migrateUniversesToDB.js');
  return migrateUniversesToDB();
}

beforeEach(() => {
  files = {};
  dirEntries = null;
  recordsByDir = {};
  typeIndex = null;
  renameShouldFail = false;
  renamed = [];
  uTable = new Map();
  rTable = new Map();
  query.mockClear();
});

describe('migrateUniversesToDB', () => {
  it('fresh install (no legacy dir): no-op WITHOUT a marker (keeps the recovery re-import hatch open)', async () => {
    const result = await importer();
    expect(result).toMatchObject({ ok: true, reason: 'fresh-install', imported: 0 });
    // No marker stamped — a later restore of data/universes.imported → data/universes
    // must re-import on the next boot rather than being silently suppressed.
    expect(MARKER in files).toBe(false);
    expect(uTable.size).toBe(0);
  });

  it('marker already present: no-op (does not re-walk)', async () => {
    files[MARKER] = JSON.stringify({ reason: 'imported' });
    dirEntries = ['u-1'];
    recordsByDir['/fake/data/universes/u-1/index.json'] = rec('u-1');
    const result = await importer();
    expect(result).toMatchObject({ reason: 'already-applied' });
    expect(uTable.size).toBe(0); // never touched the table
  });

  it('imports records + runs verbatim, then renames the dir aside and stamps marker', async () => {
    dirEntries = ['u-1', 'u-2', 'index.json', '.DS_Store'];
    recordsByDir['/fake/data/universes/u-1/index.json'] = rec('u-1', { logline: 'a' });
    recordsByDir['/fake/data/universes/u-2/index.json'] = rec('u-2');
    typeIndex = { config: { runs: [
      { id: 'r-1', universeId: 'u-1', jobIds: [], promptCount: 3, createdAt: '2026-01-03T00:00:00.000Z' },
    ] } };
    const result = await importer();
    expect(result).toMatchObject({ ok: true, reason: 'imported', imported: 2, runs: 1 });
    expect(uTable.get('u-1').data.logline).toBe('a'); // verbatim
    expect(rTable.has('r-1')).toBe(true);
    expect(renamed).toEqual([['/fake/data/universes', '/fake/data/universes.imported']]);
    expect(MARKER in files).toBe(true);
  });

  it('imports a record holding a lone-surrogate \\ud83d escape, repaired, instead of retaining it for retry', async () => {
    dirEntries = ['u-1'];
    recordsByDir['/fake/data/universes/u-1/index.json'] = rec('u-1', { logline: 'cut\ud83d', credit: '\ud83d\ude00 kept' });
    const result = await importer();
    expect(result).toMatchObject({ ok: true, reason: 'imported', imported: 1 });
    expect(uTable.get('u-1').data).toMatchObject({ logline: 'cut\uFFFD', credit: '😀 kept' });
    expect(renamed).toEqual([['/fake/data/universes', '/fake/data/universes.imported']]);
    expect(MARKER in files).toBe(true);
  });

  it('is idempotent: a re-run over the same rows imports 0 (ON CONFLICT DO NOTHING)', async () => {
    dirEntries = ['u-1'];
    recordsByDir['/fake/data/universes/u-1/index.json'] = rec('u-1');
    uTable.set('u-1', { id: 'u-1' }); // already in the table from a prior partial run
    const result = await importer();
    expect(result.imported).toBe(0);
    expect(renamed.length).toBe(1); // still renames aside + stamps marker
    expect(MARKER in files).toBe(true);
  });

  it('skips a record whose index.json is missing/unparseable', async () => {
    dirEntries = ['u-1', 'u-bad'];
    recordsByDir['/fake/data/universes/u-1/index.json'] = rec('u-1');
    recordsByDir['/fake/data/universes/u-bad/index.json'] = null; // readJSONFile → null
    const result = await importer();
    expect(result.imported).toBe(1);
    expect(uTable.has('u-1')).toBe(true);
    expect(uTable.has('u-bad')).toBe(false);
  });

  it('rename failure leaves the marker UNwritten so next boot retries', async () => {
    dirEntries = ['u-1'];
    recordsByDir['/fake/data/universes/u-1/index.json'] = rec('u-1');
    renameShouldFail = true;
    const result = await importer();
    expect(result).toMatchObject({ ok: true, reason: 'imported-rename-failed', imported: 1 });
    expect(MARKER in files).toBe(false); // no marker → retry next boot
    expect(uTable.has('u-1')).toBe(true); // rows already landed; retry is safe (ON CONFLICT)
  });
});