Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Share-bucket subscriptions.
*
* A subscription is a persistent (bucketId, recordKind, recordId) tuple. It
* says "as long as this exists, I am continuously sharing this record into
* that bucket — every local edit auto-re-exports onto a deterministic file
* in the bucket, replacing the previous snapshot."
*
* Per the user mental model: clicking a bucket in ShareToButton creates a
* subscription + does the initial export. Clicking again removes it (and
* deletes the bucket-side file so recipients see the share go away). Local
* mutations to the subscribed record fire `recordEvents` which this service
* listens to and re-exports via the same deterministic filename.
*
* Subscriptions are local-only state (data/sharing/subscriptions.json) — the
* bucket itself only holds the deterministic manifest file. Multiple peers
* may both be subscribing the same bucket; each writes their own
* sub-<kind>-<recordId>-<senderInstanceId>.json with their own source
* attribution, so two peers sharing the same record don't collide.
*/
import { join } from 'path';
import { unlink } from 'fs/promises';
import { existsSync } from 'fs';
import { PATHS, atomicWrite, readJSONFile, ensureDir } from '../../lib/fileUtils.js';
import { createFileWriteQueue } from '../../lib/fileWriteQueue.js';
import { getBucket } from './buckets.js';
import { getSeries } from '../pipeline/series.js';
import { getUniverse } from '../universeBuilder.js';
import { exportSeries, exportUniverse } from './exporter.js';
import {
recordEvents,
isReexportSuppressed,
withReexportSuppressed,
__resetReexportSuppression,
} from './recordEvents.js';
// Re-export so existing callers (importer.js etc.) keep their import path
// stable; the canonical implementation now lives in recordEvents.js.
export { withReexportSuppressed };
import { subscriptionFilename, legacySubscriptionFilename } from './manifest.js';
import { getInstanceId } from '../instanceIdentity.js';
import { isStr } from '../../lib/textUtils.js';
import { codedError } from '../../lib/codedError.js';
// Re-export from the canonical source so other modules can import the
// filename helper from either side without forcing a manifest.js import.
export { subscriptionFilename };
export const SUBSCRIBABLE_KINDS = Object.freeze(['series', 'universe']);
export const ERR_NOT_FOUND = 'SHARING_SUBSCRIPTION_NOT_FOUND';
export const ERR_VALIDATION = 'SHARING_SUBSCRIPTION_VALIDATION';
export const ERR_DUPLICATE = 'SHARING_SUBSCRIPTION_DUPLICATE';
const queueStateWrite = createFileWriteQueue();
const STATE_PATH = () => join(PATHS.data, 'sharing', 'subscriptions.json');
function subId({ bucketId, recordKind, recordId }) {
return `sub-${recordKind}-${recordId}-${bucketId}`;
}
async function readState() {
await ensureDir(join(PATHS.data, 'sharing'));
const raw = await readJSONFile(STATE_PATH(), { subscriptions: [] }, { logError: false, strict: true });
const subs = Array.isArray(raw.subscriptions) ? raw.subscriptions : [];
return { subscriptions: subs };
}
async function writeState(state) {
await ensureDir(join(PATHS.data, 'sharing'));
await atomicWrite(STATE_PATH(), state);
}
export async function listSubscriptions(filter = {}) {
const { subscriptions } = await readState();
return subscriptions.filter((s) => {
if (filter.bucketId && s.bucketId !== filter.bucketId) return false;
if (filter.recordKind && s.recordKind !== filter.recordKind) return false;
if (filter.recordId && s.recordId !== filter.recordId) return false;
return true;
});
}
export async function findSubscription(bucketId, recordKind, recordId) {
if (!bucketId || !recordKind || !recordId) return null;
const { subscriptions } = await readState();
return subscriptions.find(
(s) => s.bucketId === bucketId && s.recordKind === recordKind && s.recordId === recordId,
) || null;
}
/**
* Adopt an inbound subscription as a local outgoing subscription without
* immediately exporting back into the bucket. Importing a subscribed record
* means the local user has joined that collaboration, so ShareToButton should
* show the source bucket as checked and future local edits should re-export.
*/
export async function adoptImportedSubscription({ bucketId, recordKind, recordId, lastManifestId = null }) {
if (!SUBSCRIBABLE_KINDS.includes(recordKind)) return null;
if (!isStr(bucketId) || !isStr(recordId)) return null;
await getBucket(bucketId);
return queueStateWrite(async () => {
const state = await readState();
const id = subId({ bucketId, recordKind, recordId });
const now = new Date().toISOString();
let sub = state.subscriptions.find((s) => s.id === id);
if (!sub) {
sub = {
id,
bucketId,
recordKind,
recordId,
createdAt: now,
updatedAt: now,
lastManifestId,
lastExportedAt: null,
adoptedFromImport: true,
};
state.subscriptions.push(sub);
} else {
sub.updatedAt = now;
sub.lastManifestId = lastManifestId || sub.lastManifestId || null;
}
await writeState(state);
return sub;
});
}
/**
* Create a subscription and trigger the first export. If a subscription
* already exists for the same (bucket, kind, id), idempotently re-export
* rather than throwing — clicking "subscribe" on an already-subscribed
* bucket should be a no-op refresh, not an error.
*/
export async function subscribe({ bucketId, recordKind, recordId }) {
if (!SUBSCRIBABLE_KINDS.includes(recordKind)) {
throw codedError(`subscribable kinds are ${SUBSCRIBABLE_KINDS.join(', ')} (got "${recordKind}")`, ERR_VALIDATION);
}
if (!isStr(bucketId) || !isStr(recordId)) {
throw codedError('bucketId and recordId are required', ERR_VALIDATION);
}
// Validate the bucket exists (throws ERR_NOT_FOUND otherwise).
await getBucket(bucketId);
const id = subId({ bucketId, recordKind, recordId });
const sub = await queueStateWrite(async () => {
const state = await readState();
let live = state.subscriptions.find((s) => s.id === id);
if (!live) {
const now = new Date().toISOString();
live = { id, bucketId, recordKind, recordId, createdAt: now, updatedAt: now, lastManifestId: null, lastExportedAt: null };
state.subscriptions.push(live);
await writeState(state);
}
return live;
});
// Export outside the state queue (serialized per subscription), then stamp
// only a still-live row.
return enqueueExport(id, async () => {
const exp = await runExport(sub);
return exp ? stampExport(sub, exp) : sub;
});
}
// subId → tail of that subscription's export chain. Exports of one
// subscription run strictly one after another so the last manifest written
// is always the newest content, and unsubscribe can wait out a running one.
const inFlightExports = new Map();
function enqueueExport(id, task) {
const prev = inFlightExports.get(id) || Promise.resolve();
const run = prev.then(task);
const tail = run.catch(() => {});
inFlightExports.set(id, tail);
tail.then(() => {
if (inFlightExports.get(id) === tail) inFlightExports.delete(id);
});
return run;
}
async function isSubscriptionLive(id) {
const { subscriptions } = await queueStateWrite(readState);
return subscriptions.some((s) => s.id === id);
}
/** Internal: dispatch to the right exporter based on recordKind. */
async function runExport({ id, bucketId, recordKind, recordId }) {
const opts = { subscription: { recordKind, recordId }, isLive: () => isSubscriptionLive(id) };
if (recordKind === 'series') return exportSeries(recordId, bucketId, opts);
if (recordKind === 'universe') return exportUniverse(recordId, bucketId, opts);
return null;
}
export async function unsubscribe(id) {
if (!isStr(id)) throw codedError('subscription id required', ERR_VALIDATION);
const sub = await queueStateWrite(async () => {
const state = await readState();
const idx = state.subscriptions.findIndex((s) => s.id === id);
if (idx < 0) throw codedError(`Subscription not found: ${id}`, ERR_NOT_FOUND);
const [removed] = state.subscriptions.splice(idx, 1);
await writeState(state);
return removed;
});
// Cancel any pending debounced re-export — otherwise the timer fires ~3s
// later, the sub is gone, and reexportNow no-ops, but the timer + the
// reference to its closure linger in the pendingTimers Map until the
// setTimeout callback runs.
const pending = pendingTimers.get(sub.id);
if (pending) {
clearTimeout(pending);
pendingTimers.delete(sub.id);
}
// An export already running would otherwise write its manifest after the
// delete below and resurrect the share. Its liveness check now fails, but
// wait for it (and any queued follow-ups) so the delete always lands last.
await inFlightExports.get(sub.id);
// Delete the deterministic file from the bucket. Best-effort: the bucket
// path may be unmounted; the subscription record removal still wins so the
// user's "I want to stop sharing" intent is honored regardless. Importers
// on other peers see the file vanish via chokidar `unlink`.
//
// Also clean up the pre-sharing-v2 legacy filename (`sub-<kind>-<id>.json`)
// if it exists AND was authored by THIS instance — a peer that subscribed
// pre-upgrade and unsubscribed post-upgrade would otherwise leave a phantom
// file in the bucket. Read the file's `senderInstanceId` before unlinking
// so we don't stomp on a different peer's pre-upgrade share. We only unlink
// on a strict positive match — a legacy file missing `senderInstanceId`
// (very early v1 or malformed) is left alone, since ownership cannot be
// verified and removing a peer's share is worse than leaving a phantom.
const bucket = await getBucket(sub.bucketId).catch(() => null);
if (bucket) {
const myInstanceId = await getInstanceId().catch(() => null);
const filename = subscriptionFilename({ ...sub, senderInstanceId: myInstanceId });
const filePath = join(bucket.path, 'manifests', filename);
if (existsSync(filePath)) {
await unlink(filePath).catch((err) => {
console.log(`⚠️ sharing.subscriptions: failed to remove ${filePath}: ${err.message}`);
});
}
const legacyName = legacySubscriptionFilename(sub);
const legacyPath = join(bucket.path, 'manifests', legacyName);
if (existsSync(legacyPath) && isStr(myInstanceId)) {
// Read-only ownership check before unlink; never saves a fallback document.
const legacy = await readJSONFile(legacyPath, null, { logError: false });
if (legacy && legacy.senderInstanceId === myInstanceId) {
await unlink(legacyPath).catch((err) => {
console.log(`⚠️ sharing.subscriptions: failed to remove legacy ${legacyPath}: ${err.message}`);
});
}
}
}
return { id, removed: true };
}
/**
* Drop every subscription that targets a given (recordKind, recordId). Used
* by the `deleted` recordEvents listener so deleting a series/universe
* automatically tears down its outgoing subscriptions instead of leaving
* orphaned entries that fail every re-export.
*/
export async function unsubscribeAllForRecord(recordKind, recordId) {
const matching = await listSubscriptions({ recordKind, recordId });
const removed = [];
for (const sub of matching) {
await unsubscribe(sub.id).catch((err) => {
console.log(`⚠️ sharing.subscriptions: auto-unsubscribe failed for ${sub.id}: ${err.message}`);
});
removed.push(sub.id);
}
return { removed };
}
/**
* Re-export every subscription for a given (recordKind, recordId). Called by
* the recordEvents listener. Debounced per subscription so a flurry of edits
* coalesces into one export per ~3s window.
*/
const DEBOUNCE_MS = 3000;
const pendingTimers = new Map(); // subId → Timeout
function reexportNow(sub) {
return enqueueExport(sub.id, async () => {
const exp = await runExport(sub).catch((err) => {
console.log(`⚠️ sharing.subscriptions: re-export failed for ${sub.id}: ${err.message}`);
return null;
});
if (exp) await stampExport(sub, exp);
});
}
function stampExport(sub, exp) {
return queueStateWrite(async () => {
const state = await readState();
const live = state.subscriptions.find((s) => s.id === sub.id);
if (!live) return null; // unsubscribed while the export was running
live.lastManifestId = exp.manifestId;
live.lastExportedAt = new Date().toISOString();
live.updatedAt = live.lastExportedAt;
await writeState(state);
return live;
});
}
export async function reexportSubscribedRecord(recordKind, recordId) {
if (isReexportSuppressed(recordKind, recordId)) return;
const subs = await listSubscriptions({ recordKind, recordId });
for (const sub of subs) {
const existing = pendingTimers.get(sub.id);
if (existing) clearTimeout(existing);
const t = setTimeout(() => {
pendingTimers.delete(sub.id);
reexportNow(sub).catch(() => {});
}, DEBOUNCE_MS);
// unref so a pending debounce can't keep the process alive past shutdown.
if (typeof t.unref === 'function') t.unref();
pendingTimers.set(sub.id, t);
}
}
async function loadRecordUpdatedAt(recordKind, recordId) {
const record = recordKind === 'series' ? await getSeries(recordId)
: recordKind === 'universe' ? await getUniverse(recordId)
: null;
return isStr(record?.updatedAt) ? record.updatedAt : null;
}
/**
* Boot convergence for share buckets. The 3 s re-export debounce lives only in
* memory, so an edit made just before a restart/crash never reaches the bucket.
* Re-export every subscription whose record changed after its last export (or
* was never exported) through the same runExport + stampExport path the
* debounce uses. A subscription adopted from an import has no export yet; its
* baseline is the adoption time so an unedited imported record is not written
* straight back to the bucket it came from.
*/
export async function reconcileSubscriptionsOnBoot() {
const subscriptions = await listSubscriptions();
let reexported = 0;
for (const sub of subscriptions) {
const baseline = sub.lastExportedAt || (sub.adoptedFromImport ? sub.updatedAt : null);
const recordUpdatedAt = await loadRecordUpdatedAt(sub.recordKind, sub.recordId).catch(() => undefined);
// undefined = record missing/unreadable (nothing to export); null = no timestamp.
if (recordUpdatedAt === undefined) continue;
const stale = baseline
? recordUpdatedAt !== null && recordUpdatedAt > baseline
: true;
if (!stale) continue;
await reexportNow(sub);
reexported += 1;
}
if (reexported > 0) console.log(`🔄 sharing.subscriptions: boot reconcile re-exported ${reexported} stale share${reexported === 1 ? '' : 's'}`);
return { checked: subscriptions.length, reexported };
}
// Track the listener handles we install so __resetForTests can remove just
// ours — `removeAllListeners('updated')` would also kill listeners installed
// by neighboring test files that share the recordEvents bus.
let onUpdated = null;
let onDeleted = null;
/** Attach the recordEvents listeners — call once during sharing init. */
export function installSubscriptionListener() {
if (onUpdated || onDeleted) return;
onUpdated = ({ recordKind, recordId }) => {
reexportSubscribedRecord(recordKind, recordId).catch((err) => {
console.log(`⚠️ sharing.subscriptions: listener error for ${recordKind}/${recordId}: ${err.message}`);
});
};
onDeleted = ({ recordKind, recordId }) => {
unsubscribeAllForRecord(recordKind, recordId).catch((err) => {
console.log(`⚠️ sharing.subscriptions: delete-listener error for ${recordKind}/${recordId}: ${err.message}`);
});
};
recordEvents.on('updated', onUpdated);
recordEvents.on('deleted', onDeleted);
}
/**
* Test-only: stop new re-exports (debounce timers, listeners) and wait for the
* ones already running. A fixture must await this before removing its bucket,
* or a late export recreates the directory after removal (#12288).
*/
export async function __settleForTests() {
__resetForTests();
while (inFlightExports.size) await Promise.all([...inFlightExports.values()]);
await queueStateWrite(async () => {});
}
/** Test-only: clear pending debounces and detach our own listeners. */
export function __resetForTests() {
for (const t of pendingTimers.values()) clearTimeout(t);
pendingTimers.clear();
__resetReexportSuppression();
if (onUpdated) recordEvents.off('updated', onUpdated);
if (onDeleted) recordEvents.off('deleted', onDeleted);
onUpdated = null;
onDeleted = null;
}