Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Tests for the legacy data/pipeline-series/{id}/index.json → Postgres importer
* (#1015). Unlike the universe importer, this renames each record's index.json
* aside IN PLACE (not the whole dir) so the file-primary manuscript-review.json
* siblings stay readable at their canonical path. Disk + PG are mocked.
*/
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { readFile, stat, readdir, rename } from 'fs/promises';
import { posixPath as toPosix } from '../lib/testHelper.js';
let files = {}; // path → string (markers)
let dirEntries = null; // readdir(legacyDir) result, or null = ENOENT
let recordsByDir = {}; // `${legacyDir}/${id}/index.json` → record object
let renamed = [];
vi.mock('../lib/fileUtils.js', () => ({
PATHS: { data: '/fake/data' },
readJSONFile: vi.fn(async (path) => recordsByDir[toPosix(path)] ?? null),
}));
// Marker I/O is delegated to migrationMarker.js; mock it against the same
// in-memory `files` map (the marker helper has its own unit test).
vi.mock('../lib/migrationMarker.js', () => ({
markerExists: vi.fn(async (name) => (`/fake/data/${name}` in files)),
writeMarker: vi.fn(async (name, payload) => { files[`/fake/data/${name}`] = JSON.stringify(payload, null, 2); }),
}));
vi.mock('fs/promises', () => ({
readFile: vi.fn(async (path) => {
const key = toPosix(path);
if (!(key in recordsByDir)) { const error = new Error('ENOENT'); error.code = 'ENOENT'; throw error; }
return JSON.stringify(recordsByDir[key]);
}),
rename: vi.fn(async (from, to) => { renamed.push([toPosix(from), toPosix(to)]); }),
readdir: vi.fn(async () => (dirEntries === null ? [] : dirEntries)),
stat: vi.fn(async (path) => {
if (toPosix(path) === '/fake/data/pipeline-series') {
if (dirEntries === null) { const e = new Error('ENOENT'); e.code = 'ENOENT'; throw e; }
return { isDirectory: () => true };
}
const e = new Error('ENOENT'); e.code = 'ENOENT'; throw e;
}),
}));
let table;
const query = vi.fn(async (sql, params) => {
if (/INSERT INTO pipeline_series/.test(sql)) {
const id = params[0];
if (table.has(id)) return { rowCount: 0, rows: [] };
table.set(id, { id, name: params[1], universeId: params[2], data: JSON.parse(params[4]) });
return { rowCount: 1, rows: [] };
}
return { rowCount: 0, rows: [] };
});
vi.mock('../lib/db.js', () => ({ query: (...a) => query(...a) }));
const MARKER = '/fake/data/pipeline-series.migrated.json';
const rec = (id, extra = {}) => ({ id, name: id, createdAt: '2026-01-01T00:00:00.000Z', updatedAt: '2026-01-02T00:00:00.000Z', ...extra });
async function importer() {
const { migrateSeriesToDB } = await import('./migrateSeriesToDB.js');
return migrateSeriesToDB();
}
beforeEach(() => {
files = {};
dirEntries = null;
recordsByDir = {};
renamed = [];
table = new Map();
query.mockClear();
});
describe('migrateSeriesToDB', () => {
it('fresh install (no legacy dir): no-op WITHOUT a marker', async () => {
const result = await importer();
expect(result).toMatchObject({ ok: true, reason: 'fresh-install', imported: 0 });
expect(MARKER in files).toBe(false);
expect(table.size).toBe(0);
});
it('marker already present: no-op (does not re-walk)', async () => {
files[MARKER] = JSON.stringify({ reason: 'imported' });
dirEntries = ['ser-1'];
recordsByDir['/fake/data/pipeline-series/ser-1/index.json'] = rec('ser-1');
const result = await importer();
expect(result).toMatchObject({ reason: 'already-applied' });
expect(table.size).toBe(0);
});
it('imports records verbatim, renames each index.json aside IN PLACE, stamps marker', async () => {
dirEntries = ['ser-1', 'ser-2', 'index.json', '.DS_Store'];
recordsByDir['/fake/data/pipeline-series/ser-1/index.json'] = rec('ser-1', { universeId: 'u-1', logline: 'a' });
recordsByDir['/fake/data/pipeline-series/ser-2/index.json'] = rec('ser-2');
const result = await importer();
expect(result).toMatchObject({ ok: true, reason: 'imported', imported: 2 });
expect(table.get('ser-1').data.logline).toBe('a'); // verbatim
expect(table.get('ser-1').universeId).toBe('u-1'); // mirror column
// Per-record index.json renamed aside — dir itself NOT renamed (siblings stay).
expect(renamed).toContainEqual(['/fake/data/pipeline-series/ser-1/index.json', '/fake/data/pipeline-series/ser-1/index.json.imported']);
expect(renamed).toContainEqual(['/fake/data/pipeline-series/ser-2/index.json', '/fake/data/pipeline-series/ser-2/index.json.imported']);
// The whole-dir rename the universe importer does must NOT happen here.
expect(renamed.some(([from]) => from === '/fake/data/pipeline-series')).toBe(false);
expect(MARKER in files).toBe(true);
});
it('is idempotent: a re-run over the same rows imports 0 (ON CONFLICT DO NOTHING)', async () => {
dirEntries = ['ser-1'];
recordsByDir['/fake/data/pipeline-series/ser-1/index.json'] = rec('ser-1');
table.set('ser-1', { id: 'ser-1' });
const result = await importer();
expect(result.imported).toBe(0);
expect(MARKER in files).toBe(true);
});
it('skips non-record entries and unreadable records', async () => {
dirEntries = ['ser-1', 'ser-bad', 'index.json'];
recordsByDir['/fake/data/pipeline-series/ser-1/index.json'] = rec('ser-1');
recordsByDir['/fake/data/pipeline-series/ser-bad/index.json'] = null;
const result = await importer();
expect(result.imported).toBe(1);
expect(table.has('ser-1')).toBe(true);
expect(table.has('ser-bad')).toBe(false);
});
});
it.each(['stat', 'readdir'])('surfaces unexpected %s failures without marking a completed import', async operation => {
dirEntries = ['ser-a'];
const error = Object.assign(new Error('access denied'), { code: 'EACCES' });
({ stat, readdir })[operation].mockRejectedValueOnce(error);
await expect(importer()).rejects.toThrow('access denied');
expect(MARKER in files).toBe(false);
});
it.each(['read', 'park'])('retries after a source %s failure without falsely completing', async operation => {
dirEntries = ['ser-a'];
recordsByDir['/fake/data/pipeline-series/ser-a/index.json'] = rec('ser-a');
const error = Object.assign(new Error('access denied'), { code: 'EACCES' });
(operation === 'read' ? readFile : rename).mockRejectedValueOnce(error);
expect(await importer()).toMatchObject({ reason: 'incomplete' });
expect(MARKER in files).toBe(false);
expect(await importer()).toMatchObject({ reason: 'imported' });
expect(MARKER in files).toBe(true);
});