Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Postgres-backed round-trip for the data-broker database + case ledger
* (issue #2144). Skips cleanly when no DB is reachable (CI, fresh checkout);
* runs the seed/refresh/ledger surface when one is. Cleans up only the auto
* (non-curated) brokers + cases it creates — the curated seed is idempotent and
* left in place. Runs via `npm run test:db` (→ portos_test) ONLY; db.js guards
* refuse writes to a non-test DB.
*/
import { describe, it, expect, afterAll, beforeAll } from 'vitest';
import { checkHealth, ensureSchema, query, close } from '../lib/db.js';
import { requireDbOrSkip } from '../lib/dbTestGate.js';
import { mkdtempSync, writeFileSync, rmSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
// A valid vault key BEFORE privacyVault is imported so the runScanPass test can
// create a scan-eligible name without touching the repo's real .env.
const originalKey = process.env.PRIVACY_VAULT_KEY;
process.env.PRIVACY_VAULT_KEY = 'd'.repeat(64);
let dbReady = false;
let skipReason = '';
{
const health = await checkHealth().catch((e) => ({ connected: false, error: e?.message }));
if (!health.connected) {
skipReason = `Postgres not reachable (${health.error || 'no connection'})`;
} else {
await ensureSchema().catch(() => {});
const probe = await query(
`SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'privacy_brokers') AS ok`,
).catch(() => ({ rows: [{ ok: false }] }));
if (probe.rows?.[0]?.ok) dbReady = true;
else skipReason = 'privacy_brokers table not present';
}
}
const runDb = requireDbOrSkip('services/privacyBrokers.db.test', dbReady, skipReason);
describe.skipIf(!runDb)('privacy brokers DB round-trip', () => {
let svc;
let scan;
let vault;
const autoBrokerIds = [
'test-auto-alpha', 'ca-test-beta-inc', 'test-auto-upgrade',
'test-evidence-seal', 'test-evidence-erase', 'test-evidence-legacy', 'test-evidence-vault',
];
const createdVaultIds = [];
const _vaultTableWasEmpty = false;
const testStart = new Date().toISOString();
beforeAll(async () => {
svc = await import('./privacyBrokers.js');
scan = await import('./privacyScan.js');
vault = await import('./privacyVault.js');
// The scan pass needs an explicit broker_scan grant (#8332) — the seeded
// `self` row holds only local-vault consent. Removed in afterAll.
const subjects = await import('./privacySubjects.js');
await subjects.recordConsent({ scope: 'broker_scan', method: 'self', note: 'privacyBrokers.db.test' });
});
afterAll(async () => {
for (const id of createdVaultIds) {
await query(`DELETE FROM privacy_vault_records WHERE id = $1`, [id]).catch(() => {});
}
await query(`DELETE FROM privacy_consents WHERE scope IN ('pii_vault', 'broker_scan') AND granted_at >= $1`, [testStart]).catch(() => {});
for (const id of autoBrokerIds) {
await query(`DELETE FROM privacy_broker_cases WHERE broker_id = $1`, [id]).catch(() => {});
await query(`DELETE FROM privacy_brokers WHERE id = $1`, [id]).catch(() => {});
}
// Drop the curated brokers' cases this suite created so a re-run starts clean.
await query(`DELETE FROM privacy_broker_cases`).catch(() => {});
await close();
if (originalKey === undefined) delete process.env.PRIVACY_VAULT_KEY;
else process.env.PRIVACY_VAULT_KEY = originalKey;
});
it('seeds the curated brokers idempotently with the cluster wired up', async () => {
const first = await svc.seedCuratedBrokers();
const { rows: c1 } = await query(`SELECT COUNT(*)::int AS n FROM privacy_brokers WHERE source = 'curated'`);
await svc.seedCuratedBrokers(); // second run must not duplicate
const { rows: c2 } = await query(`SELECT COUNT(*)::int AS n FROM privacy_brokers WHERE source = 'curated'`);
expect(first.seeded).toBeGreaterThan(0);
expect(c2[0].n).toBe(c1[0].n);
const parent = await svc.getBroker('peopleconnect');
expect(parent.preferSuppression).toBe(true);
const child = await svc.getBroker('truthfinder');
expect(child.clusterParent).toBe('peopleconnect');
});
it('ensureSeeded updates curated brokers on version upgrade while preserving user settings and auto sources (#4019)', async () => {
svc.resetEnsureSeededForTests();
// 1. Disable a curated broker to verify user-modified setting is preserved
await svc.setBrokerEnabled('spokeo', false);
// 2. Insert a dummy non-curated broker to verify auto sources are preserved
await query(
`INSERT INTO privacy_brokers (id, name, urls, optout, tier, disclosure_fields, source, confidence, enabled, created_at, updated_at)
VALUES ('test-auto-upgrade', 'Auto Upgrade', '{}', '{}', 2, '{}', 'badbool', 'auto', true, NOW(), NOW())
ON CONFLICT (id) DO NOTHING`,
);
// 3. Mutate a curated broker's field in DB to simulate an older version record
await query(`UPDATE privacy_brokers SET name = 'Old Spokeo Name' WHERE id = 'spokeo'`);
// 4. Trigger ensureSeeded
svc.resetEnsureSeededForTests();
await svc.ensureSeeded();
// 5. Verify curated broker definition is updated from seed file (name restored)
const spokeo = await svc.getBroker('spokeo');
expect(spokeo.name).toBe('Spokeo');
// Verify user-modified enabled setting (false) was preserved
expect(spokeo.enabled).toBe(false);
// 6. Verify non-curated broker was untouched
const autoBroker = await svc.getBroker('test-auto-upgrade');
expect(autoBroker).toMatchObject({ name: 'Auto Upgrade', source: 'badbool' });
// Cleanup
await svc.setBrokerEnabled('spokeo', true);
});
it('refresh adds auto brokers and NEVER clobbers a curated row', async () => {
const before = await svc.getBroker('spokeo');
expect(before.source).toBe('curated');
const res = await svc.refreshBrokers({
// A collision on the curated `spokeo` id (must be ignored) + one genuinely
// new auto broker.
fetchBadbool: async () => [
{ id: 'spokeo', name: 'HIJACKED', source: 'badbool', confidence: 'auto' },
{ id: 'test-auto-alpha', name: 'Test Auto Alpha', url: 'https://alpha.example' },
],
fetchCaRegistry: async () => [
{ id: 'ca-test-beta-inc', name: 'Beta Inc', urls: { home: 'https://beta.example' }, source: 'ca_registry', confidence: 'auto' },
],
});
expect(res.added).toBe(2);
const afterSpokeo = await svc.getBroker('spokeo');
expect(afterSpokeo.name).toBe(before.name); // curated name preserved
expect(afterSpokeo.source).toBe('curated');
const auto = await svc.getBroker('test-auto-alpha');
expect(auto).toMatchObject({ name: 'Test Auto Alpha', source: 'badbool', confidence: 'auto' });
});
it('records a scan verdict, stamps next_recheck_at, and enforces the state machine', async () => {
const now = new Date('2026-07-08T00:00:00.000Z');
// Fresh verdict on a curated broker.
const kase = await svc.recordScanVerdict('test-auto-alpha', 'found', {
evidence: { match_basis: 'name+location', listing_urls: ['https://alpha.example/jane'] },
found: true, now,
});
expect(kase.state).toBe('found');
// found → +1 day recheck.
expect(new Date(kase.nextRecheckAt).toISOString()).toBe('2026-07-09T00:00:00.000Z');
// rowToCase serializes the server-derived legal manual moves (issue #2417);
// a found case can always be queued as a human task, never re-stamped itself.
expect(kase.allowedTransitions).toContain('human_task_queued');
expect(kase.allowedTransitions).not.toContain('found');
// Lifecycle forward via transitionCase.
const submitted = await svc.transitionCase(kase.id, 'optout_in_progress');
expect(submitted.state).toBe('optout_in_progress');
const s2 = await svc.transitionCase(kase.id, 'submitted', { channel: 'web_form', now });
expect(s2.state).toBe('submitted');
expect(new Date(s2.nextRecheckAt).toISOString()).toBe('2026-07-11T00:00:00.000Z'); // +3d
// confirmed_removed is refused from a submission path (no rescan).
await expect(svc.transitionCase(kase.id, 'confirmed_removed'))
.rejects.toMatchObject({ code: 'CONFIRMED_REQUIRES_RESCAN' });
// Reach awaiting_processing, then confirm via a verifying rescan.
await svc.transitionCase(kase.id, 'verification_pending');
await svc.transitionCase(kase.id, 'awaiting_processing');
const removed = await svc.transitionCase(kase.id, 'confirmed_removed', { viaRescan: true, now });
expect(removed.state).toBe('confirmed_removed');
expect(new Date(removed.nextRecheckAt).toISOString()).toBe('2026-08-07T00:00:00.000Z'); // +30d
});
it('a manual transition onto a verdict state syncs the ledger found flag', async () => {
const now = new Date('2026-07-08T00:00:00.000Z');
// ca-test-beta-inc: its case is untouched by the lifecycle test above
// (test-auto-alpha's case ends at confirmed_removed, which blocked can't
// be recorded over).
const kase = await svc.recordScanVerdict('ca-test-beta-inc', 'blocked', {
evidence: { match_basis: 'antibot_wall', search_url: 'https://beta.example/jane' }, now,
});
expect(kase.found).toBe(null);
// The blocked-case "I'm listed" manual verdict implies found: true.
const confirmed = await svc.transitionCase(kase.id, 'found');
expect(confirmed.found).toBe(true);
// And a manual not_found implies found: false; explicit patch still wins.
const dismissed = await svc.transitionCase(kase.id, 'not_found');
expect(dismissed.found).toBe(false);
});
it('runScanPass scans due brokers, records verdicts, and skips opt-out-owned cases', async () => {
// A scan-eligible name so buildSearchVectors is non-empty.
const nameRec = await vault.createVaultRecord({ type: 'legal_name', label: 'Legal name', value: 'Jane Q Publictest' });
createdVaultIds.push(nameRec.id);
const addrRec = await vault.createVaultRecord({ type: 'address', label: 'Home', value: '1 Oak Ave, Portland, OR 97201' });
createdVaultIds.push(addrRec.id);
// Every broker's static page returns a no-match body → not_found verdicts.
const fetchImpl = async () => ({ status: 200, text: async () => 'No results found for that search. '.repeat(40) });
const summary = await scan.runScanPass({ fetchImpl, browserFetch: async () => null, urlSafe: async () => true });
expect(summary.scanned).toBeGreaterThan(0);
expect(summary.verdicts.not_found).toBeGreaterThan(0);
// The confirmed_removed case (opt-out-owned) from the previous test must be
// untouched — the pass never overwrites it with a raw scan verdict.
const owned = await svc.getCaseForBroker('test-auto-alpha');
expect(owned.state).toBe('confirmed_removed');
});
// ─── Sealed identity evidence (#8333) ─────────────────────────────────────
// Fake identity values, distinctive enough that a substring hit on the raw
// JSONB column (what pg_dump writes) can only mean a plaintext copy.
const IDENTITY = {
matched_name: 'Jane Q Sealtest',
matched_location: 'Sealville',
search_url: 'https://seal-broker.example/Jane-Sealtest/Sealville',
listing_urls: ['https://seal-broker.example/p/jane-sealtest'],
};
const insertTestBroker = (id) => query(
`INSERT INTO privacy_brokers (id, name, source, confidence, enabled, created_at, updated_at)
VALUES ($1, $1, 'badbool', 'auto', FALSE, NOW(), NOW()) ON CONFLICT (id) DO NOTHING`, [id],
);
const rawEvidence = async (caseId) => (await query(
`SELECT evidence::text AS t FROM privacy_broker_cases WHERE id = $1`, [caseId],
)).rows[0].t;
const expectNoPlaintextIdentity = (text) => {
for (const needle of ['Sealtest', 'Sealville', 'seal-broker.example']) expect(text).not.toContain(needle);
};
it('stores scan identity evidence sealed, reveals it on request, and erases it on confirmed_removed', async () => {
await insertTestBroker('test-evidence-seal');
const kase = await svc.recordScanVerdict('test-evidence-seal', 'found', {
evidence: { match_basis: 'name+location', ...IDENTITY }, found: true,
});
// At rest (and so in every dump): no plaintext name, location, or URL.
expectNoPlaintextIdentity(await rawEvidence(kase.id));
// The API projection carries the verdict metadata + a summary, never the identity.
expect(kase.evidence).toEqual({ match_basis: 'name+location' });
expect(kase.identityEvidence).toEqual({ sealed: true, listingCount: 1 });
expect((await svc.revealCaseEvidence(kase.id)).evidence).toEqual(IDENTITY);
// Lifecycle transitions that restate workflow metadata carry the envelope forward.
await svc.transitionCase(kase.id, 'optout_in_progress', { evidence: { lane: 'web_form' } });
await svc.transitionCase(kase.id, 'submitted');
const pending = await svc.transitionCase(kase.id, 'verification_pending');
expect(pending.identityEvidence).toEqual({ sealed: true, listingCount: 1 });
expect((await svc.revealCaseEvidence(kase.id)).evidence.search_url).toBe(IDENTITY.search_url);
// The verifying re-scan's confirmed_removed erases the identity copy but
// keeps the non-identifying evidence, the state, and the case history.
const removed = await svc.transitionCase(kase.id, 'confirmed_removed', { viaRescan: true });
expect(removed.state).toBe('confirmed_removed');
expect(removed.identityEvidence).toBe(null);
expect(removed.evidence).toEqual({ lane: 'web_form' });
expect(new Date(removed.createdAt).getTime()).toBe(new Date(kase.createdAt).getTime());
expect(await rawEvidence(kase.id)).not.toContain('sealed_identity');
expect(await svc.revealCaseEvidence(kase.id)).toEqual({ caseId: kase.id, sealed: false, evidence: {} });
});
it('erases one case\'s identity evidence on request, keeping its state and verdict', async () => {
await insertTestBroker('test-evidence-erase');
const kase = await svc.recordScanVerdict('test-evidence-erase', 'blocked', {
evidence: { match_basis: 'antibot_wall', search_url: IDENTITY.search_url },
});
const cleared = await svc.clearCaseIdentityEvidence(kase.id);
expect(cleared).toMatchObject({ state: 'blocked', evidence: { match_basis: 'antibot_wall' }, identityEvidence: null });
// Gone from later API responses too.
const listed = (await svc.listBrokerCases()).find((c) => c.id === kase.id);
expect(listed.identityEvidence).toBe(null);
expect((await svc.revealCaseEvidence(kase.id)).evidence).toEqual({});
await expect(svc.clearCaseIdentityEvidence('00000000-0000-4000-8000-00000000dead')).rejects.toMatchObject({ status: 404 });
});
it('converts a legacy plaintext case idempotently, and a failed conversion changes nothing', async () => {
await insertTestBroker('test-evidence-legacy');
const legacyId = '00000000-0000-4000-8000-0000000833aa';
await query(
`INSERT INTO privacy_broker_cases (id, broker_id, state, found, evidence, next_recheck_at, created_at, updated_at)
VALUES ($1, 'test-evidence-legacy', 'found', TRUE, $2, NOW(), NOW(), NOW())`,
[legacyId, JSON.stringify({ match_basis: 'name+location', ...IDENTITY })],
);
const before = await rawEvidence(legacyId);
// No usable key and an unwritable .env: the conversion cannot seal, so the
// read that triggered it fails and the row is left exactly as it was.
const vaultCrypto = await import('../lib/vaultCrypto.js');
const blockerDir = mkdtempSync(join(tmpdir(), 'portos-8333-'));
writeFileSync(join(blockerDir, 'not-a-dir'), '');
const savedKey = process.env.PRIVACY_VAULT_KEY;
delete process.env.PRIVACY_VAULT_KEY;
vaultCrypto.__setVaultEnvPathForTests(join(blockerDir, 'not-a-dir', '.env'));
svc.__resetCaseEvidenceSealForTests();
try {
await expect(svc.listBrokerCases()).rejects.toThrow();
expect(await rawEvidence(legacyId)).toBe(before);
} finally {
process.env.PRIVACY_VAULT_KEY = savedKey;
vaultCrypto.__setVaultEnvPathForTests(null);
rmSync(blockerDir, { recursive: true, force: true });
}
// With the key back, the next read converts before returning anything.
const listed = (await svc.listBrokerCases()).find((c) => c.id === legacyId);
expect(listed.evidence).toEqual({ match_basis: 'name+location' });
expect(listed.identityEvidence).toEqual({ sealed: true, listingCount: 1 });
expectNoPlaintextIdentity(await rawEvidence(legacyId));
expect((await svc.revealCaseEvidence(legacyId)).evidence).toEqual(IDENTITY);
// Idempotent: a second conversion pass leaves the sealed row byte-identical
// (it is not re-encrypted, which would mint a new IV).
const sealed = await rawEvidence(legacyId);
svc.__resetCaseEvidenceSealForTests();
await svc.listBrokerCases();
expect(await rawEvidence(legacyId)).toBe(sealed);
});
it('deleting the source vault record leaves no identity copy in the case API or the stored row', async () => {
await insertTestBroker('test-evidence-vault');
const nameRec = await vault.createVaultRecord({ type: 'legal_name', label: 'Seal test name', value: IDENTITY.matched_name });
const kase = await svc.recordScanVerdict('test-evidence-vault', 'found', {
evidence: { match_basis: 'name+location', ...IDENTITY }, found: true,
});
expect(kase.identityEvidence?.sealed).toBe(true);
await vault.deleteVaultRecord(nameRec.id);
const after = await svc.getCaseForBroker('test-evidence-vault');
expect(after.state).toBe('found');
expect(after.identityEvidence).toBe(null);
expect(after.evidence).toEqual({ match_basis: 'name+location' });
const raw = await rawEvidence(kase.id);
expectNoPlaintextIdentity(raw);
expect(raw).not.toContain('sealed_identity');
});
it('reports scan status counts', async () => {
const status = await svc.getScanStatus();
expect(status.enabledBrokers).toBeGreaterThan(0);
expect(typeof status.caseCounts).toBe('object');
expect(typeof status.dueForRecheck).toBe('number');
});
});