Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Per-category sync integrity: build a local manifest and compare it against
* a remote peer's manifest to surface records that are out-of-parity.
*
* `buildLocalManifest(kind)` — returns one row per record with id, name,
* updatedAt, deleted, and sorted sha256 asset hashes. Tombstones are
* included so that deletes diff correctly against the peer.
*
* `getPeerIntegrity({ peerId, kind })` — fetches the peer's manifest via
* GET /api/peer-sync/manifest, then runs the pure diff. Returns
* `{ available: bool, reason?, records: [...] }`.
*/
import { computeRecordIntegrity } from '../../lib/syncIntegrity.js';
import { listCollections } from '../mediaCollections.js';
import { listUniverses } from '../universeBuilder.js';
import { listSeries } from '../pipeline/series.js';
import { listLooms } from '../fableLoom/index.js';
import { getPeers } from '../instances.js';
import { assetIntegrityForRecord } from './peerSync.js';
import { peerBaseUrl } from '../../lib/peerUrl.js';
import { peerFetch, readPeerBody } from '../../lib/peerHttpClient.js';
const PEER_INTEGRITY_TIMEOUT_MS = 15_000;
const PEER_INTEGRITY_MAX_BYTES = 16 * 1024 * 1024;
async function recordsForKind(kind) {
if (kind === 'mediaCollection') return listCollections({ includeDeleted: true });
if (kind === 'universe') return listUniverses({ includeDeleted: true });
if (kind === 'series') return listSeries({ includeDeleted: true });
if (kind === 'fableLoom') return listLooms({ includeDeleted: true });
return [];
}
/**
* Build a local manifest for the given kind.
* One row per record: `{ id, name, updatedAt, deleted, assetHashes }`.
* Includes tombstoned records so deletes surface correctly in the diff.
*
* @param {'universe'|'series'|'mediaCollection'|'fableLoom'} kind
* @returns {Promise<Array>}
*/
export async function buildLocalManifest(kind) {
const records = await recordsForKind(kind);
// Hash records SEQUENTIALLY (for...of, not Promise.all(map)) so a large
// library can't fan out an unbounded number of concurrent file-hash reads and
// spike CPU/disk. Each assetShaListForRecord already reads many files; doing
// every record's pass at once would multiply that.
const out = [];
for (const r of records) {
const deleted = r.deleted === true;
const assetIntegrity = deleted
? { assetHashes: [], metadataMissing: false }
: await assetIntegrityForRecord(kind, r);
out.push({
id: r.id,
name: r.name,
updatedAt: r.updatedAt,
deleted,
// Tombstones never need asset hashes: computeRecordIntegrity only
// compares assetHashes when BOTH sides are live (and drops
// deleted-vs-deleted pairs entirely). Hashing a deleted record's
// still-on-disk assets is pure wasted file I/O.
assetHashes: assetIntegrity.assetHashes,
metadataMissing: assetIntegrity.metadataMissing,
});
}
return out;
}
/**
* Fetch the peer's manifest for `kind`, run the local-vs-remote diff, and
* return the classified record list.
*
* @param {{ peerId: string, kind: string }} opts
* @returns {Promise<{ available: boolean, reason?: string, records: Array }>}
*/
export async function getPeerIntegrity({ peerId, kind }) {
const peers = await getPeers().catch(() => []);
const peer = peers.find((p) => p.instanceId === peerId) || null;
if (!peer) return { available: false, reason: 'peer-not-found', records: [] };
const res = await peerFetch(
`${peerBaseUrl(peer)}/api/peer-sync/manifest?kind=${encodeURIComponent(kind)}`,
{ signal: AbortSignal.timeout(PEER_INTEGRITY_TIMEOUT_MS), maxBytes: PEER_INTEGRITY_MAX_BYTES },
peer,
).catch(() => null);
// Distinguish a network failure (peerFetch threw / returned null) from a 404.
// A null result means the peer is offline/unreachable — NOT that it's running
// an older PortOS without the /manifest route. Lumping them together would
// tell the user "peer too old, upgrade it" when the peer is simply down.
if (!res) return { available: false, reason: 'peer-unreachable', records: [] };
if (res.status === 404) return { available: false, reason: 'peer-too-old', records: [] };
if (!res.ok) return { available: false, reason: 'fetch-failed', records: [] };
const body = await readPeerBody(res, 'json', {
idleTimeoutMs: PEER_INTEGRITY_TIMEOUT_MS,
maxBytes: PEER_INTEGRITY_MAX_BYTES,
}).catch(() => null);
if (!body) return { available: false, reason: 'fetch-failed', records: [] };
// The peer response is untrusted — computeRecordIntegrity assumes every entry
// is a non-null object with a string `id` (it reads `r.id` and keys a Map on
// it). A hostile/malformed manifest (nulls, scalars, id-less objects) would
// otherwise throw and 500 this endpoint, so filter to well-formed rows first.
const remote = (Array.isArray(body?.records) ? body.records : []).filter(
(r) => r && typeof r === 'object' && typeof r.id === 'string' && r.id,
);
const local = await buildLocalManifest(kind);
return { available: true, records: computeRecordIntegrity(local, remote) };
}