Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Federated peer-sync — standalone media-library sync (#1566).
*
* Advertises a library-level manifest (every generated image/video, pipeline
* audio, uploaded music) at `GET /api/peer-sync/library-manifest`, and (for
* full-sync peers) diffs a peer's manifest against local disk + pulls the
* missing bytes via the shared asset-pull primitives. Independent of the
* per-record subscription pipeline.
*
* Split out of the former 4,004-line peerSync.js (#1830).
*/
import { join } from 'path';
import { readdir, stat } from 'fs/promises';
import { createHash } from 'crypto';
import { PATHS } from '../../lib/fileUtils.js';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { createFileWriteQueue } from '../../lib/fileWriteQueue.js';
import { isPlainObject } from '../../lib/objects.js';
import { peerBaseUrl } from '../../lib/peerUrl.js';
import { peerFetch, readPeerBody } from '../../lib/peerHttpClient.js';
import { RESPONSE_TOO_LARGE } from '../../lib/httpClient.js';
import { withAbortTimeout } from '../../lib/abortTimeout.js';
import { sanitizeAssetFilename } from './buckets.js';
import { PORTOS_SCHEMA_VERSIONS } from '../../lib/schemaVersions.js';
import { getPeers } from '../instances.js';
import { peerLibraryManifestSchema } from '../../lib/validation.js';
import {
hashSimpleAsset,
hashImageForManifest,
diffAssetManifestAgainstLocal,
pullMissingAssetsFromPeer,
ASSET_PULL_TIMEOUT_MS,
} from './peerSyncAssets.js';
import { findPeerById, FORCE_REVALIDATE_EVERY } from './peerSyncShared.js';
import { isStr, isNonBlankStr } from '../../lib/textUtils.js';
// --- Standalone media-library federation (#1566) ------------------------
//
// The per-record pipeline above replicates only the bytes a SYNCED creative
// record references. For a declared full-sync peer pair we ALSO mirror the
// STANDALONE media library — every generated image/video, pipeline audio, and
// user-uploaded music — so each peer's Media tab is a complete replica.
//
// Shape: the sender advertises a library-level manifest (filename + sha256 per
// asset) at GET /api/peer-sync/library-manifest; a receiver (only for peers it
// has flagged fullSync) fetches it, diffs vs local disk via the SAME
// diffAssetManifestAgainstLocal + pullOneAsset machinery the per-record path
// uses, and rebuilds the derived media_assets index once bytes land. Notably:
// - video THUMBNAILS are regenerated locally on each video pull
// (doPullOneAsset's video branch), not byte-federated;
// - video-history METADATA already union-merges via the `videoHistory`
// dataSync category — the bytes are what this adds;
// - the generic data/history.jsonl action log is machine-local and never
// federated (it's app activity, not media gen history).
// The on-disk media kinds the library manifest covers, resolved at CALL TIME
// (not frozen at module load) so a redirected PATHS — the test-suite tmpdir
// pattern, and consistent with directoryForAssetKind reading PATHS live — is
// honored. `image` carries a gen-params sidecar (rides via hashImageForManifest);
// the rest are flat bytes. image-refs are EXCLUDED (ephemeral FLUX multi-ref
// scratch); video-thumbnails are EXCLUDED (regenerated locally on video pull).
//
// Exported because this list — not MEDIA_LIBRARY_KIND_DIRNAMES below, which
// only translates kinds into backup exclude patterns — is what actually
// decides which `data/` directories cross to a peer. The #28 federation guard
// asserts against it directly, so a machine-local media kind cannot be added
// to the walk unnoticed.
export function mediaLibraryDirs() {
return [
{ kind: 'image', dir: PATHS.images },
{ kind: 'video', dir: PATHS.videos },
{ kind: 'audio', dir: PATHS.audio },
{ kind: 'music', dir: PATHS.music },
];
}
// Stable data-root-relative basename per kind, used only to match backup
// exclude patterns. Independent of PATHS so a redirected path can't change which
// exclude pattern applies.
const MEDIA_LIBRARY_KIND_DIRNAMES = Object.freeze({
image: 'images', video: 'videos', audio: 'audio', music: 'music',
});
// Cap so a pathologically large library can't build an unbounded manifest. 100k
// assets is far beyond any realistic single-user library; when exceeded we LOG
// and truncate (per AGENTS.md "no silent caps") rather than ship an open-ended
// list. Pagination is a clean follow-up if ever hit. Kept in sync with the
// `assets` array cap in peerLibraryManifestSchema.
const MEDIA_LIBRARY_MANIFEST_CAP = 100_000;
/**
* Pure matcher: given a list of effective rsync exclude patterns, return the Set
* of media-library KINDS the user has excluded from backup (and therefore from
* federation, per the #1566 acceptance criterion). Checked at DIRECTORY
* granularity — recognizes a whole-dir exclude (`/videos`, `/videos/`,
* `/videos/**`, or the bare `videos/` form). Per-file glob granularity is out of
* scope: none of the media dirs are excluded by DEFAULT_EXCLUDES, so this only
* fires on a custom user exclude like `/music/`, and excluding individual files
* from federation isn't a supported control.
*
* Exported for unit testing without mocking the settings/backup IO.
*/
export function libraryKindsExcludedByPatterns(effectiveExcludes) {
const excluded = new Set();
const patterns = Array.isArray(effectiveExcludes) ? effectiveExcludes : [];
// Normalize each pattern to its bare anchored segment in one pass: strip
// leading slashes, a trailing `/**`/`/*`, or a trailing slash → `/videos/**`
// and `videos/` both collapse to `videos`.
const normalized = patterns.map((p) => String(p).replace(/^\/+|\/+\*+$|\/+$/g, ''));
for (const [kind, name] of Object.entries(MEDIA_LIBRARY_KIND_DIRNAMES)) {
if (normalized.includes(name)) excluded.add(kind);
}
return excluded;
}
// Honor the backup exclusion contract (#1566 acceptance). Best-effort: a
// settings/backup read failure federates everything (the prior behavior), never
// throws. Composes the IO (read settings, compute effective excludes) with the
// pure `libraryKindsExcludedByPatterns` matcher above.
async function excludedLibraryKinds() {
const settingsMod = await import('../settings.js').catch(() => null);
const backupMod = await import('../backup.js').catch(() => null);
if (!settingsMod?.getSettings || !backupMod?.computeEffectiveExcludes) return new Set();
const settings = await settingsMod.getSettings().catch(() => null);
const excludePaths = Array.isArray(settings?.backup?.excludePaths) ? settings.backup.excludePaths : [];
const disabledDefaultExcludes = Array.isArray(settings?.backup?.disabledDefaultExcludes) ? settings.backup.disabledDefaultExcludes : [];
const effective = backupMod.computeEffectiveExcludes({ excludePaths, disabledDefaultExcludes });
return libraryKindsExcludedByPatterns(effective);
}
// In-memory hash cache for the flat (non-image) library kinds, keyed by full
// path → { mtimeMs, size, entry }. The manifest is rebuilt on every poll (each
// full-sync peer fetches it ~every 60s), and video/music files can be large —
// re-`sha256File`-ing a multi-GB library every poll is real, avoidable disk I/O.
// Images already cache their sha in the sidecar (getOrComputeImageSha256), so
// this only covers video/audio/music. Invalidated on (mtimeMs, size) change —
// the same cheap signal the image sidecar cache uses; a re-render writes a new
// file (new mtime), so staleness isn't a concern.
const libraryFlatHashCache = new Map(); // fullPath → { mtimeMs, size, entry }
async function hashCachedLibraryAsset(name, kind, dir) {
const safeName = sanitizeAssetFilename(name);
if (!safeName) return null;
const fullPath = join(dir, safeName);
const st = await stat(fullPath).catch(() => null);
if (!st || !st.isFile()) return null;
const cached = libraryFlatHashCache.get(fullPath);
if (cached && cached.mtimeMs === st.mtimeMs && cached.size === st.size) return cached.entry;
const entry = await hashSimpleAsset(safeName, kind, dir);
if (entry) libraryFlatHashCache.set(fullPath, { mtimeMs: st.mtimeMs, size: st.size, entry });
return entry;
}
/**
* Build the standalone media-library manifest this instance advertises to
* full-sync peers. Walks each media dir, hashes every file (reusing the
* per-record hashers so the wire shape is identical), and stamps a `manifestHash`
* over the sorted entries so a receiver can short-circuit an unchanged library.
*
* @returns {Promise<{ schemaVersion:number, manifestHash:string, assets:Array }>}
*/
export async function buildMediaLibraryManifest() {
const excluded = await excludedLibraryKinds();
const assets = [];
let truncated = false;
for (const { kind, dir } of mediaLibraryDirs()) {
if (truncated) break;
if (excluded.has(kind)) continue;
const names = await readdir(dir).catch(() => []); // missing dir → empty (nothing of that kind yet)
for (const name of names) {
// Image sidecars are metadata, not standalone assets — they ride the image
// entry's sidecarSha256 + the receiver's pullSidecarForImage, so skip the
// `.json` files in the images dir.
if (kind === 'image' && name.endsWith('.json')) continue;
const entry = kind === 'image'
? await hashImageForManifest(name) // sidecar-cached
: await hashCachedLibraryAsset(name, kind, dir); // (mtime,size)-cached
if (!entry) continue;
if (assets.length >= MEDIA_LIBRARY_MANIFEST_CAP) { truncated = true; break; }
assets.push(entry);
}
}
if (truncated) {
console.log(`⚠️ peerSync: media-library manifest hit the ${MEDIA_LIBRARY_MANIFEST_CAP}-asset cap — truncating (some assets won't federate; pagination is a follow-up)`);
}
// Deterministic order (sort by filename) so the manifestHash converges across
// machines regardless of readdir order / filesystem.
const sorted = assets.sort((a, b) => (a.filename < b.filename ? -1 : a.filename > b.filename ? 1 : 0));
const manifestHash = createHash('sha256')
.update(sorted.map((e) => `${e.kind}:${e.filename}:${e.sha256 || ''}:${e.sidecarSha256 || ''}`).join('\n'))
.digest('hex');
return { schemaVersion: PORTOS_SCHEMA_VERSIONS.mediaLibrary, manifestHash, assets: sorted };
}
// Receiver-side: last manifestHash fully processed per peer, so an unchanged
// library skips the diff entirely. In-memory (rebuilt on restart — the first
// post-boot sweep just re-confirms disk, cheap because the diff finds everything
// present and pulls nothing).
const lastLibraryManifestHash = new Map(); // peerInstanceId → manifestHash
// Landed bytes stay durable when the derived index is unavailable. Keep repair
// pending independently of the byte diff so an empty diff can retry it.
const libraryIndexRepairPending = new Set(); // peerInstanceId
// Consecutive unchanged-manifest ticks per peer; after FORCE_REVALIDATE_EVERY we
// force a full re-diff even though the remote manifest is unchanged, so a local
// file loss self-heals without waiting for a restart or a remote library change.
const libraryUnchangedSkips = new Map(); // peerInstanceId → count
// Per-peer re-entrancy guard so a slow sweep (large pull) can't overlap itself
// when the periodic tick fires again before it finishes.
const librarySweepInFlight = new Set(); // peerInstanceId
// The manifest JSON itself (not the bytes — those ride the per-asset 100MB cap).
const MEDIA_LIBRARY_MANIFEST_MAX_BYTES = 32 * 1024 * 1024;
// `reconcileMediaAssets` rebuilds the WHOLE media_assets index from disk, so two
// sweeps finishing close together would fire overlapping full-table upsert/delete
// passes against the same rows (#3929). Serialize on one tail AND collapse
// requests that arrive while a pass is still QUEUED — one pass reading disk after
// the last pull settles already covers every caller. `reconcilePending` clears the
// moment the work starts, so a request that arrives mid-pass gets its own pass
// chained behind (it may have landed bytes the running pass already read past).
const reconcileQueue = createFileWriteQueue();
let reconcilePending = null;
function reconcileMediaLibraryIndex() {
if (reconcilePending) return reconcilePending;
reconcilePending = reconcileQueue(async () => {
reconcilePending = null;
// Dynamic import keeps the DB-backed index module out of peerSync's static
// graph (it no-ops under the file/test backend). image+video rows are rebuilt
// from disk; audio/music aren't indexed (served from disk directly).
const mod = await import('../mediaAssetIndex/index.js').catch(() => null);
if (!mod?.reconcileMediaAssets) return false;
// The pull wrote bytes outside any lease; indexing them takes one, so a cut
// that copied the media directories first never dumps a row for a file it missed.
const result = await withBackupAssetPublication(() => mod.reconcileMediaAssets({ requireComplete: true })).catch((err) => {
console.error(`❌ peerSync: media_assets reconcile after library sweep failed: ${err.message}`);
return null;
});
return result?.ok === true && Array.isArray(result.skippedPrune) && result.skippedPrune.length === 0;
});
return reconcilePending;
}
/**
* Receiver-pull the standalone media library from ONE full-sync peer: fetch its
* manifest, gate on schema version, diff vs local disk, pull missing bytes, then
* rebuild the derived media_assets index. No-op for a non-full-sync peer.
*
* Best-effort + idempotent: every guard returns rather than throws so a periodic
* caller can fire it unconditionally.
*
* @param {object} peer a peer entry from getPeers()
* @returns {Promise<{ pulled:number, skipped?:string }>}
*/
export async function syncMediaLibraryFromPeer(peer) {
if (!isPlainObject(peer) || peer.fullSync !== true || !isStr(peer.instanceId)) {
return { pulled: 0, skipped: 'not-fullsync' };
}
if (librarySweepInFlight.has(peer.instanceId)) return { pulled: 0, skipped: 'in-flight' };
librarySweepInFlight.add(peer.instanceId);
try {
const url = `${peerBaseUrl(peer)}/api/peer-sync/library-manifest`;
const res = await withAbortTimeout(ASSET_PULL_TIMEOUT_MS, (signal) =>
peerFetch(url, { signal, maxBytes: MEDIA_LIBRARY_MANIFEST_MAX_BYTES }, peer))
.catch(() => null);
if (!res || !res.ok) return { pulled: 0, skipped: 'unreachable' };
// Enforce the manifest cap before buffering the body. peerFetch's `maxBytes`
// only streams-caps the HTTPS (host) shim; for a plain-HTTP (address) peer it
// delegates to native fetch, which ignores `maxBytes`, so `res.json()` would
// otherwise buffer an unbounded body. Express on the sender sets Content-Length
// for the JSON response, so the declared-length check is a fast path; a chunked
// body that declares none is capped while streaming by readPeerBody below.
const declaredLen = Number(res.headers?.get?.('content-length'));
if (Number.isFinite(declaredLen) && declaredLen > MEDIA_LIBRARY_MANIFEST_MAX_BYTES) {
console.log(`⚠️ peerSync: media-library manifest from ${peer.name || peer.instanceId} too large (${declaredLen} > ${MEDIA_LIBRARY_MANIFEST_MAX_BYTES}) — skipping`);
return { pulled: 0, skipped: 'too-large' };
}
let tooLarge = false;
const body = await readPeerBody(res, 'json', { maxBytes: MEDIA_LIBRARY_MANIFEST_MAX_BYTES })
.catch((err) => {
tooLarge = err?.code === RESPONSE_TOO_LARGE;
return null;
});
if (tooLarge) {
console.log(`⚠️ peerSync: media-library manifest from ${peer.name || peer.instanceId} streamed past ${MEDIA_LIBRARY_MANIFEST_MAX_BYTES} bytes — skipping`);
return { pulled: 0, skipped: 'too-large' };
}
const parsed = peerLibraryManifestSchema.safeParse(body);
if (!parsed.success) {
console.log(`⚠️ peerSync: media-library manifest from ${peer.name || peer.instanceId} failed validation — skipping`);
return { pulled: 0, skipped: 'invalid' };
}
const manifest = parsed.data;
// Schema gate — GENTLE skip (not reject): the sender's manifest envelope is
// newer than this instance understands. Wait for the local PortOS to upgrade
// rather than mis-pull against a contract we can't read. (Bytes are
// version-agnostic, but a manifest-SHAPE bump means a new field we'd mishandle.)
if (manifest.schemaVersion > PORTOS_SCHEMA_VERSIONS.mediaLibrary) {
console.log(`⏸️ peerSync: ${peer.name || peer.instanceId} media-library manifest is schema v${manifest.schemaVersion} > local v${PORTOS_SCHEMA_VERSIONS.mediaLibrary} — skipping until this instance updates`);
return { pulled: 0, skipped: 'schema-ahead' };
}
// Unchanged-library short-circuit — but force a full re-diff every
// FORCE_REVALIDATE_EVERY consecutive unchanged ticks so a LOCAL file loss /
// corruption self-heals even while the REMOTE manifest stays put. (The
// recorded hash is in-memory, so a process restart also re-diffs; this covers
// the mid-session window between restarts.)
if (!libraryIndexRepairPending.has(peer.instanceId) && lastLibraryManifestHash.get(peer.instanceId) === manifest.manifestHash) {
const skips = (libraryUnchangedSkips.get(peer.instanceId) || 0) + 1;
if (skips < FORCE_REVALIDATE_EVERY) {
libraryUnchangedSkips.set(peer.instanceId, skips);
return { pulled: 0, skipped: 'unchanged' };
}
libraryUnchangedSkips.set(peer.instanceId, 0); // periodic forced re-diff — fall through
}
const missing = await diffAssetManifestAgainstLocal(manifest.assets);
const requested = missing.length;
let stillMissing = missing;
if (requested > 0) {
// Reuse the per-record pull worker (in-flight dedup, image-sidecar fetch,
// video-thumbnail regen). It swallows per-asset failures, so re-diff disk
// rather than treating the pull's resolution as evidence bytes landed.
await pullMissingAssetsFromPeer(peer.instanceId, missing);
stillMissing = await diffAssetManifestAgainstLocal(manifest.assets);
}
const pulled = requested - stillMissing.length;
if (pulled > 0) libraryIndexRepairPending.add(peer.instanceId);
if (libraryIndexRepairPending.has(peer.instanceId) && await reconcileMediaLibraryIndex()) {
libraryIndexRepairPending.delete(peer.instanceId);
}
if (stillMissing.length === 0 && !libraryIndexRepairPending.has(peer.instanceId)) {
// Full sweep, including index repair — safe to short-circuit future ticks on this manifestHash.
lastLibraryManifestHash.set(peer.instanceId, manifest.manifestHash);
if (pulled > 0) console.log(`📥 peerSync: media-library sweep from ${peer.name || peer.instanceId} — pulled ${pulled} asset(s)`);
} else if (stillMissing.length > 0) {
// Partial pull — do NOT record the hash, so the next tick re-diffs and
// retries the still-missing assets instead of being marked done.
console.log(`⚠️ peerSync: media-library sweep from ${peer.name || peer.instanceId} — pulled ${pulled}/${requested}, ${stillMissing.length} still missing; retrying next tick`);
}
return requested > 0 ? { pulled, missing: stillMissing.length } : { pulled };
} finally {
librarySweepInFlight.delete(peer.instanceId);
}
}
// Global (all-peers) re-entrancy guard — distinct from the per-peer
// `librarySweepInFlight` above. That set only stops ONE peer's sweep from
// overlapping itself; a sweep that outlives the 60s tick would still let tick N+1
// start peer B while tick N is mid-pull on peer A. Two full-sync peers commonly
// advertise the SAME filenames (that is what full-sync means), so overlapping
// per-peer sweeps are exactly the cross-peer download + reconcile race in #3929.
// One library sweep at a time; the next tick picks up wherever this one left off.
let allPeersSweepInFlight = false;
/**
* Periodic driver: sweep the standalone media library from every full-sync peer.
* Called on a timer from initSharing. Peers are swept one at a time and
* best-effort; the per-peer re-entrancy guard + manifestHash short-circuit keep
* an unchanged library cheap.
*
* @returns {Promise<{ peers:number, skipped?:string }>}
*/
export async function syncMediaLibraryWithAllPeers() {
if (allPeersSweepInFlight) return { peers: 0, skipped: 'in-flight' };
allPeersSweepInFlight = true;
try {
const peers = await getPeers().catch(() => []);
const fullSyncPeers = peers.filter((p) => p?.fullSync === true && p?.enabled !== false && isStr(p.instanceId));
for (const peer of fullSyncPeers) {
await syncMediaLibraryFromPeer(peer).catch((err) => {
console.log(`⚠️ peerSync: media-library sweep for ${peer.name || peer.instanceId} failed: ${err.message}`);
});
}
return { peers: fullSyncPeers.length };
} finally {
allPeersSweepInFlight = false;
}
}