Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Unit tests for catalogIngestSources — the url / file / voice ingest paths.
* Every external boundary is mocked (browser CDP, Whisper, DB, extraction,
* disk) so the test stays pure. Asserts:
* - URL ingest fetches main text, creates a 'url' scrap with url metadata,
* and runs the extraction pipeline.
* - File ingest records filename/mime on a 'file' scrap.
* - Voice ingest transcribes, persists audio to mint a media_key, and stamps
* it onto a 'voice-memo' scrap; an empty transcript / empty audio throws
* BEFORE any scrap or audio file is written.
* - fetchUrlMainText throws when the page yields no readable text.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
vi.mock('./catalogDB.js', () => ({
createScrap: vi.fn(),
createChunkedScrap: vi.fn(),
}));
vi.mock('./catalogExtraction.js', () => ({
extractIngredients: vi.fn(),
extractIngredientsForScrap: vi.fn(),
}));
vi.mock('./brainStorage.js', () => ({
getIdeaById: vi.fn(),
getMemoryEntryById: vi.fn(),
getProjectById: vi.fn(),
getAdminById: vi.fn(),
getPersonById: vi.fn(),
}));
vi.mock('./voice/stt.js', () => ({
transcribe: vi.fn(),
}));
vi.mock('./browserService.js', () => ({
navigateToUrlPinned: vi.fn(),
closeCdpPage: vi.fn(),
}));
vi.mock('fs/promises', () => ({
writeFile: vi.fn(),
}));
vi.mock('dns/promises', () => ({
lookup: vi.fn(),
}));
vi.mock('../lib/fileUtils.js', () => ({
PATHS: { audio: '/tmp/data/audio' },
ensureDir: vi.fn(),
sleep: vi.fn(() => Promise.resolve()),
writeFileGuarded: vi.fn().mockResolvedValue(undefined),
safeJSONParse: vi.fn((raw, fallback) => {
try {
return JSON.parse(raw);
} catch {
return fallback;
}
}),
}));
const catalogDB = await import('./catalogDB.js');
const catalogExtraction = await import('./catalogExtraction.js');
const browserService = await import('./browserService.js');
const brainStorage = await import('./brainStorage.js');
const _fsp = await import('fs/promises');
const dnsp = await import('dns/promises');
const fileUtils = await import('../lib/fileUtils.js');
const {
fetchUrlMainText,
ingestFromUrl,
ingestFromFile,
ingestFromVoice,
ingestFromBrain,
brainRecordToIngestText,
} = await import('./catalogIngestSources.js');
const DRAFT = { runId: 'r1', characters: [], stages: [] };
beforeEach(() => {
vi.clearAllMocks();
catalogExtraction.extractIngredientsForScrap.mockResolvedValue(DRAFT);
catalogDB.createChunkedScrap.mockImplementation(async (args) => ({ id: 'cat-scrap-1', ...args }));
// Hostnames resolve to a safe public address by default (the DNS SSRF guard).
dnsp.lookup.mockResolvedValue({ address: '93.184.216.34', family: 4 });
// Default: Chrome dials a single public IP for the document (no rebind), and
// the pinned DOM read returns a simple article.
browserService.navigateToUrlPinned.mockImplementation(pinnedNav({
finalUrl: 'https://ex.com/a',
hops: ['93.184.216.34'],
evalResult: JSON.stringify({ title: 'Article A', text: 'Body text here.' }),
}));
});
/**
* Faithfully mimic the real `navigateToUrlPinned` fail-closed contract: it
* verifies EVERY main-frame hop's ACTUAL Chrome-reported connection IP (the
* initial nav, each redirect, and any client-side navigation during the settle
* window) via the caller's `verifyRemoteIp` predicate, and throws (tearing the
* tab down) if any hop is refused. `hops` are the IPs Chrome actually dialed — a
* rebinding DNS answer surfaces here as a private IP the pre-nav dns.lookup never
* saw. On success it returns the DOM-read result (`evalResult`) it performed on
* the SAME pinned session — the real function reads the DOM itself, so no
* separate read is mocked. It owns the settle wait, so it ignores `settleMs`.
*/
function pinnedNav({ finalUrl, hops, evalResult = null }) {
return async (url, { verifyRemoteIp } = {}) => {
for (const ip of hops) {
if (!verifyRemoteIp(ip)) {
throw new Error(`refusing to ingest: Chrome connected to a disallowed address ${ip} for ${url}`);
}
}
return { id: 'p1', url: finalUrl || url, title: '', webSocketDebuggerUrl: 'ws://x', evalResult };
};
}
describe('fetchUrlMainText', () => {
it('navigates + reads main text, returning title + text + finalUrl', async () => {
const out = await fetchUrlMainText('https://ex.com/a', { settleMs: 0 });
expect(out).toEqual({ text: 'Body text here.', title: 'Article A', finalUrl: 'https://ex.com/a' });
// Navigated through the PINNED path with a remote-IP verifier predicate AND
// the DOM-read expression (the read happens on the same pinned session).
expect(browserService.navigateToUrlPinned).toHaveBeenCalledWith(
'https://ex.com/a',
expect.objectContaining({ verifyRemoteIp: expect.any(Function), evaluateExpression: expect.any(String) }),
);
// The read ran on that session, so the tab is scratch — every ingested URL
// would otherwise leave one open in the user's browser. Passing
// `evaluateExpression` IS the request for that teardown (#3365), so this
// service must not close the tab itself; `navigateToUrlPinned` already did.
expect(browserService.closeCdpPage).not.toHaveBeenCalled();
});
it('throws when the page extracts no readable text', async () => {
browserService.navigateToUrlPinned.mockImplementation(pinnedNav({
finalUrl: 'https://ex.com/a', hops: ['93.184.216.34'], evalResult: JSON.stringify({ title: '', text: ' ' }),
}));
await expect(fetchUrlMainText('https://ex.com/a', { settleMs: 0 })).rejects.toThrow(/no readable text/);
expect(catalogDB.createChunkedScrap).not.toHaveBeenCalled();
});
it('refuses to ingest a hostname that resolves to a blocked address (DNS SSRF) before navigating', async () => {
// Schema passes (it's a normal hostname), but DNS points it at cloud metadata.
dnsp.lookup.mockResolvedValue({ address: '169.254.169.254', family: 4 });
await expect(fetchUrlMainText('https://evil.example/x', { settleMs: 0 })).rejects.toThrow(/resolves to a blocked/);
expect(browserService.navigateToUrlPinned).not.toHaveBeenCalled();
});
it('refuses when Chrome connects to a private IP AFTER DNS validation passed (rebinding TOCTOU)', async () => {
// Pre-nav dns.lookup answers a public IP so assertIngestUrlSafe passes — but
// Chrome re-resolves and actually dials cloud metadata. The remote-IP pin
// catches Chrome's real connection and refuses (throws) before any text is read.
dnsp.lookup.mockResolvedValue({ address: '93.184.216.34', family: 4 });
browserService.navigateToUrlPinned.mockImplementation(pinnedNav({ finalUrl: 'https://evil.example/x', hops: ['169.254.169.254'] }));
await expect(fetchUrlMainText('https://evil.example/x', { settleMs: 0 })).rejects.toThrow(/disallowed address 169\.254\.169\.254/);
expect(catalogDB.createChunkedScrap).not.toHaveBeenCalled();
});
it('refuses when a REDIRECT hop connects to a loopback address (per-hop pin)', async () => {
// First hop dials a public IP, but the redirect Chrome followed landed on a
// loopback address — the pin verifies EVERY main-frame hop, not just the last.
browserService.navigateToUrlPinned.mockImplementation(pinnedNav({ finalUrl: 'http://127.0.0.1:5555/secret', hops: ['93.184.216.34', '127.0.0.1'] }));
await expect(fetchUrlMainText('https://ex.com/a', { settleMs: 0 })).rejects.toThrow(/disallowed address 127\.0\.0\.1/);
});
it('refuses a CLIENT-SIDE navigation during the settle window that dials metadata', async () => {
// Pin passes the initial public hop, but the page meta-refreshes /
// location.replaces during the settle window to a host Chrome dials at cloud
// metadata. Because the pin stays active through settle, that later hop's real
// IP is verified too — no stale-DNS re-vet that a rebind could slip past.
browserService.navigateToUrlPinned.mockImplementation(pinnedNav({ finalUrl: 'https://evil.example/x', hops: ['93.184.216.34', '169.254.169.254'] }));
await expect(fetchUrlMainText('https://ex.com/a', { settleMs: 0 })).rejects.toThrow(/disallowed address 169\.254\.169\.254/);
});
it('clamps an oversized page body to the raw-text cap (2M chars)', async () => {
const huge = 'x'.repeat(2_500_000);
browserService.navigateToUrlPinned.mockImplementation(pinnedNav({
finalUrl: 'https://ex.com/a', hops: ['93.184.216.34'], evalResult: JSON.stringify({ title: 'Big', text: huge }),
}));
const out = await fetchUrlMainText('https://ex.com/a', { settleMs: 0 });
expect(out.text.length).toBe(2_000_000);
});
});
describe('ingestFromUrl', () => {
it('creates a url scrap with url metadata and runs extraction', async () => {
browserService.navigateToUrlPinned.mockImplementation(pinnedNav({
finalUrl: 'https://ex.com/a', hops: ['93.184.216.34'], evalResult: JSON.stringify({ title: 'Article A', text: 'Body.' }),
}));
const { scrap, draft } = await ingestFromUrl({ url: 'https://ex.com/a', settleMs: 0 });
expect(draft).toBe(DRAFT);
const createArgs = catalogDB.createChunkedScrap.mock.calls[0][0];
expect(createArgs.sourceKind).toBe('url');
expect(createArgs.rawText).toBe('Body.');
expect(createArgs.metadata).toEqual({ url: 'https://ex.com/a', title: 'Article A' });
expect(catalogExtraction.extractIngredientsForScrap).toHaveBeenCalledWith(
expect.objectContaining({ scrapId: scrap.id }),
);
});
});
describe('ingestFromFile', () => {
it('records filename + mime on a file scrap', async () => {
const { draft } = await ingestFromFile({ text: 'Notes body', filename: 'notes.md', mime: 'text/markdown' });
expect(draft).toBe(DRAFT);
const createArgs = catalogDB.createChunkedScrap.mock.calls[0][0];
expect(createArgs.sourceKind).toBe('file');
expect(createArgs.title).toBe('notes.md');
expect(createArgs.metadata).toEqual({ filename: 'notes.md', mime: 'text/markdown' });
});
it('throws on empty file text before creating a scrap', async () => {
await expect(ingestFromFile({ text: ' ', filename: 'empty.txt' })).rejects.toThrow(/no extractable text/);
expect(catalogDB.createChunkedScrap).not.toHaveBeenCalled();
});
});
describe('ingestFromVoice', () => {
const wavBase64 = Buffer.from('fake-wav-bytes').toString('base64');
it('transcribes, persists audio to mint a media_key, and stamps it on the scrap', async () => {
const transcribeFn = vi.fn().mockResolvedValue({ text: 'spoken memo text' });
const persistFn = vi.fn().mockResolvedValue('voice-memo-abc.wav');
const { draft, mediaKey } = await ingestFromVoice(
{ audioBase64: wavBase64, mimeType: 'audio/wav', title: 'My memo' },
{ transcribeFn, persistFn },
);
expect(transcribeFn).toHaveBeenCalledWith(expect.any(Buffer), { mimeType: 'audio/wav' });
expect(persistFn).toHaveBeenCalledTimes(1);
expect(mediaKey).toBe('voice-memo-abc.wav');
expect(draft).toBe(DRAFT);
const createArgs = catalogDB.createChunkedScrap.mock.calls[0][0];
expect(createArgs.sourceKind).toBe('voice-memo');
expect(createArgs.rawText).toBe('spoken memo text');
expect(createArgs.metadata).toEqual({ mediaKey: 'voice-memo-abc.wav', mimeType: 'audio/wav' });
expect(createArgs.title).toBe('My memo');
});
it('throws on an empty transcript and never persists audio or creates a scrap', async () => {
const transcribeFn = vi.fn().mockResolvedValue({ text: ' ' });
const persistFn = vi.fn();
await expect(
ingestFromVoice({ audioBase64: wavBase64 }, { transcribeFn, persistFn }),
).rejects.toThrow(/empty transcript/);
expect(persistFn).not.toHaveBeenCalled();
expect(catalogDB.createChunkedScrap).not.toHaveBeenCalled();
});
it('throws on empty audio before transcription', async () => {
const transcribeFn = vi.fn();
await expect(
ingestFromVoice({ audioBase64: '' }, { transcribeFn }),
).rejects.toThrow(/audio was empty/);
expect(transcribeFn).not.toHaveBeenCalled();
});
it('defaults the audio persister to write under PATHS.audio with a wav extension', async () => {
const transcribeFn = vi.fn().mockResolvedValue({ text: 'hi' });
const { mediaKey } = await ingestFromVoice({ audioBase64: wavBase64 }, { transcribeFn });
// Real persistFn ran: it called writeFileGuarded under the mocked PATHS.audio.
expect(fileUtils.writeFileGuarded).toHaveBeenCalledTimes(1);
expect(mediaKey).toMatch(/^voice-memo-.*\.wav$/);
const writtenPath = fileUtils.writeFileGuarded.mock.calls[0][0];
// Composed with path.join — accept either separator.
expect(writtenPath.split('\\').join('/')).toContain('/tmp/data/audio');
});
});
describe('brainRecordToIngestText', () => {
it('folds an idea record (title + oneLiner + notes) into one block', () => {
const { title, rawText, tags } = brainRecordToIngestText('ideas', {
title: 'The city remembers',
oneLiner: 'A surveillance net becomes sentient.',
notes: 'It helps the protagonist.',
tags: ['sci-fi', 'ai'],
});
expect(title).toBe('The city remembers');
expect(rawText).toBe('The city remembers\n\nA surveillance net becomes sentient.\n\nIt helps the protagonist.');
expect(tags).toEqual(['sci-fi', 'ai']);
});
it('uses content for a memory and name for a person', () => {
expect(brainRecordToIngestText('memories', { title: 'Rooftop', content: 'Rain, neon.' }).rawText)
.toBe('Rooftop\n\nRain, neon.');
const person = brainRecordToIngestText('people', { name: 'Elena', context: 'Detective.', followUps: ['call', 'verify'] });
expect(person.title).toBe('Elena');
expect(person.rawText).toBe('Elena\n\nDetective.\n\nFollow-ups: call; verify');
});
it('falls back to a default title and drops non-string tags', () => {
const out = brainRecordToIngestText('memories', { content: 'orphan note', tags: ['ok', 5, null] });
expect(out.title).toBe('Brain entry');
expect(out.rawText).toBe('orphan note');
expect(out.tags).toEqual(['ok']);
});
});
describe('ingestFromBrain', () => {
it('resolves an idea, creates a brain-bridge scrap, and runs extraction', async () => {
brainStorage.getIdeaById.mockResolvedValue({
id: 'idea-1', title: 'Neon detective', oneLiner: 'Noir in a sentient city.', tags: ['noir'],
});
const { scrap, draft } = await ingestFromBrain({ brainType: 'ideas', brainId: 'idea-1' });
expect(draft).toBe(DRAFT);
const createArgs = catalogDB.createChunkedScrap.mock.calls[0][0];
expect(createArgs.sourceKind).toBe('brain-bridge');
expect(createArgs.title).toBe('Neon detective');
expect(createArgs.rawText).toBe('Neon detective\n\nNoir in a sentient city.');
expect(createArgs.metadata).toEqual({
brainType: 'ideas', brainId: 'idea-1', brainTitle: 'Neon detective', brainTags: ['noir'],
});
expect(catalogExtraction.extractIngredientsForScrap).toHaveBeenCalledWith(
expect.objectContaining({ scrapId: scrap.id }),
);
});
it('rejects an unsupported brain type before any lookup', async () => {
await expect(ingestFromBrain({ brainType: 'links', brainId: 'x' })).rejects.toThrow(/unsupported brain type/);
expect(catalogDB.createChunkedScrap).not.toHaveBeenCalled();
});
it('throws when the brain record is missing', async () => {
brainStorage.getMemoryEntryById.mockResolvedValue(null);
await expect(ingestFromBrain({ brainType: 'memories', brainId: 'gone' })).rejects.toThrow(/not found/);
expect(catalogDB.createChunkedScrap).not.toHaveBeenCalled();
});
it('throws when the record has no text to ingest', async () => {
brainStorage.getMemoryEntryById.mockResolvedValue({ id: 'm1', title: ' ', content: '' });
await expect(ingestFromBrain({ brainType: 'memories', brainId: 'm1' })).rejects.toThrow(/no text/);
expect(catalogDB.createChunkedScrap).not.toHaveBeenCalled();
});
});